From scipy-svn at scipy.org Tue Oct 2 02:37:10 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 2 Oct 2007 01:37:10 -0500 (CDT) Subject: [Scipy-svn] r3385 - trunk Message-ID: <20071002063710.097AC39C11D@new.scipy.org> Author: jarrod.millman Date: 2007-10-02 01:36:43 -0500 (Tue, 02 Oct 2007) New Revision: 3385 Modified: trunk/TOCHANGE.txt Log: add link to style guidelines Modified: trunk/TOCHANGE.txt =================================================================== --- trunk/TOCHANGE.txt 2007-10-01 00:37:44 UTC (rev 3384) +++ trunk/TOCHANGE.txt 2007-10-02 06:36:43 UTC (rev 3385) @@ -39,6 +39,7 @@ Documentation ------------- +See http://projects.scipy.org/scipy/numpy/wiki/CodingStyleGuidelines * use new docstring format From scipy-svn at scipy.org Tue Oct 2 02:38:58 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 2 Oct 2007 01:38:58 -0500 (CDT) Subject: [Scipy-svn] r3386 - in trunk: . scipy/io/examples Message-ID: <20071002063858.E65EDC7C037@new.scipy.org> Author: jarrod.millman Date: 2007-10-02 01:38:43 -0500 (Tue, 02 Oct 2007) New Revision: 3386 Modified: trunk/TOCHANGE.txt trunk/scipy/io/examples/read_array_demo1.py Log: rest typo Modified: trunk/TOCHANGE.txt =================================================================== --- trunk/TOCHANGE.txt 2007-10-02 06:36:43 UTC (rev 3385) +++ trunk/TOCHANGE.txt 2007-10-02 06:38:43 UTC (rev 3386) @@ -40,6 +40,7 @@ ------------- See http://projects.scipy.org/scipy/numpy/wiki/CodingStyleGuidelines + * use new docstring format Modified: trunk/scipy/io/examples/read_array_demo1.py =================================================================== --- trunk/scipy/io/examples/read_array_demo1.py 2007-10-02 06:36:43 UTC (rev 3385) +++ trunk/scipy/io/examples/read_array_demo1.py 2007-10-02 06:38:43 UTC (rev 3386) @@ -1,58 +1,58 @@ -#========================================================================= -# NAME: read_array_demo1 -# -# DESCRIPTION: Examples to read 2 columns from a multicolumn ascii text -# file, skipping the first line of header. First example reads into -# 2 separate arrays. Second example reads into a single array. Data are -# then plotted. -# -# Here is the format of the file test.txt: -# -------- -# Some header to skip -# 1 2 3 -# 2 4 6 -# 3 6 9 -# 4 8 12 -# -# USAGE: -# python read_array_demo1.py -# -# PARAMETERS: -# -# DEPENDENCIES: -# matplotlib (pylab) -# test.txt -# -# -# AUTHOR: Simon J. Hook -# DATE : 09/23/2005 -# -# MODIFICATION HISTORY: -# -# COMMENT: -# -#============================================================================ - -from scipy import * -from scipy.io import read_array -from pylab import * - -def main(): - - # First example, read first and second column from ascii file. Skip first - # line of header. - # Note use of (1,-1) in lines to skip first line and then read to end of file - # Note use of (0,) in columns to pick first column, since its a tuple need trailing comma - x=read_array("test.txt",lines=(1,-1), columns=(0,)) - y=read_array("test.txt",lines=(1,-1), columns=(1,)) - - #Second example, read the file into a single arry - z=read_array("test.txt",lines=(1,-1), columns=(0,2)) - - # Plot the data - plot(x,y,'r--',z[:,0],z[:,1]) - show() - -# The one and only main function -if __name__ == "__main__": +#========================================================================= +# NAME: read_array_demo1 +# +# DESCRIPTION: Examples to read 2 columns from a multicolumn ascii text +# file, skipping the first line of header. First example reads into +# 2 separate arrays. Second example reads into a single array. Data are +# then plotted. +# +# Here is the format of the file test.txt: +# -------- +# Some header to skip +# 1 2 3 +# 2 4 6 +# 3 6 9 +# 4 8 12 +# +# USAGE: +# python read_array_demo1.py +# +# PARAMETERS: +# +# DEPENDENCIES: +# matplotlib (pylab) +# test.txt +# +# +# AUTHOR: Simon J. Hook +# DATE : 09/23/2005 +# +# MODIFICATION HISTORY: +# +# COMMENT: +# +#============================================================================ + +from scipy import * +from scipy.io import read_array +from pylab import * + +def main(): + + # First example, read first and second column from ascii file. Skip first + # line of header. + # Note use of (1,-1) in lines to skip first line and then read to end of file + # Note use of (0,) in columns to pick first column, since its a tuple need trailing comma + x=read_array("test.txt",lines=(1,-1), columns=(0,)) + y=read_array("test.txt",lines=(1,-1), columns=(1,)) + + #Second example, read the file into a single arry + z=read_array("test.txt",lines=(1,-1), columns=(0,2)) + + # Plot the data + plot(x,y,'r--',z[:,0],z[:,1]) + show() + +# The one and only main function +if __name__ == "__main__": main() From scipy-svn at scipy.org Tue Oct 2 03:40:39 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 2 Oct 2007 02:40:39 -0500 (CDT) Subject: [Scipy-svn] r3387 - trunk/scipy/lib/blas Message-ID: <20071002074039.7F4CA39C11D@new.scipy.org> Author: pearu Date: 2007-10-02 02:40:02 -0500 (Tue, 02 Oct 2007) New Revision: 3387 Added: trunk/scipy/lib/blas/fblaswrap_veclib_c.c.src Modified: trunk/scipy/lib/blas/setup.py Log: Ported veclib support from linalg to lib.blas. Added: trunk/scipy/lib/blas/fblaswrap_veclib_c.c.src =================================================================== --- trunk/scipy/lib/blas/fblaswrap_veclib_c.c.src 2007-10-02 06:38:43 UTC (rev 3386) +++ trunk/scipy/lib/blas/fblaswrap_veclib_c.c.src 2007-10-02 07:40:02 UTC (rev 3387) @@ -0,0 +1,18 @@ +#include + +//#define WRAP_F77(a) wcblas_##a##_ +#define WRAP_F77(a) w##a##_ + +/**begin repeat +#p2=c,z,c,z# +#s2=u,u,c,c# +#ctype2=complex,double complex,complex,double complex# +*/ + +void WRAP_F77(@p2 at dot@s2@)(@ctype2@ *dot at s2@, const int *N, const @ctype2@ *X, const int *incX, const @ctype2@ *Y, const int *incY) +{ + cblas_ at p2@dot at s2@_sub(*N, X, *incX, Y, *incY, dot at s2@); +} + +/**end repeat**/ + Modified: trunk/scipy/lib/blas/setup.py =================================================================== --- trunk/scipy/lib/blas/setup.py 2007-10-02 06:38:43 UTC (rev 3386) +++ trunk/scipy/lib/blas/setup.py 2007-10-02 07:40:02 UTC (rev 3387) @@ -21,6 +21,22 @@ #-------------------- +def needs_cblas_wrapper(info): + """Returns true if needs c wrapper around cblas for calling from + fortran.""" + r_accel = re.compile("Accelerate") + r_vec = re.compile("vecLib") + res = False + try: + tmpstr = info['extra_link_args'] + for i in tmpstr: + if r_accel.search(i) or r_vec.search(i): + res = True + except KeyError: + pass + + return res + tmpl_empty_cblas_pyf = ''' python module cblas usercode void empty_module(void) {} @@ -62,14 +78,19 @@ skip_names['fblas'].extend(\ 'drotmg srotmg drotm srotm'.split()) + depends = [__file__, 'fblas_l?.pyf.src', 'fblas.pyf.src','fblaswrap.f.src', + 'fblaswrap_veclib_c.c.src'] # fblas: + if needs_cblas_wrapper(blas_opt): + sources = ['fblas.pyf.src', 'fblaswrap_veclib_c.c.src'], + else: + sources = ['fblas.pyf.src','fblaswrap.f.src'] config.add_extension('fblas', - sources = ['fblas.pyf.src','fblaswrap.f.src'], - depends = [__file__,'fblas_l?.pyf.src'], + sources = sources, + depends = depends, f2py_options = ['skip:']+skip_names['fblas']+[':'], extra_info = blas_opt ) - # cblas: def get_cblas_source(ext, build_dir): name = ext.name.split('.')[-1] From scipy-svn at scipy.org Tue Oct 2 05:13:57 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 2 Oct 2007 04:13:57 -0500 (CDT) Subject: [Scipy-svn] r3388 - in trunk/scipy: cluster/tests fftpack/tests integrate/tests interpolate/tests io/tests lib/blas/tests linalg/tests linsolve/umfpack/tests maxentropy/tests ndimage/tests odr/tests optimize/tests sandbox/arpack/tests sandbox/cdavid/tests sandbox/dhuard sandbox/exmplpackage/tests sandbox/exmplpackage/yyy/tests sandbox/fdfpack/tests sandbox/maskedarray/alternative_versions sandbox/maskedarray/tests sandbox/montecarlo/tests sandbox/multigrid/tests sandbox/numexpr/tests sandbox/pyem/tests sandbox/pyloess/tests sandbox/rbf/tests sandbox/spline/tests sandbox/svm/tests sandbox/timeseries/lib/tests sandbox/timeseries/tests signal/tests sparse/tests special/tests stats/models/tests stats/tests weave/tests Message-ID: <20071002091357.6E39D39C139@new.scipy.org> Author: stefan Date: 2007-10-02 04:00:27 -0500 (Tue, 02 Oct 2007) New Revision: 3388 Modified: trunk/scipy/cluster/tests/test_vq.py trunk/scipy/fftpack/tests/test_basic.py trunk/scipy/fftpack/tests/test_helper.py trunk/scipy/fftpack/tests/test_pseudo_diffs.py trunk/scipy/integrate/tests/test_integrate.py trunk/scipy/integrate/tests/test_quadpack.py trunk/scipy/integrate/tests/test_quadrature.py trunk/scipy/interpolate/tests/test_fitpack.py trunk/scipy/interpolate/tests/test_interpolate.py trunk/scipy/io/tests/test_array_import.py trunk/scipy/io/tests/test_mio.py trunk/scipy/io/tests/test_mmio.py trunk/scipy/io/tests/test_npfile.py trunk/scipy/io/tests/test_recaster.py trunk/scipy/lib/blas/tests/test_blas.py trunk/scipy/linalg/tests/test_basic.py trunk/scipy/linalg/tests/test_blas.py trunk/scipy/linalg/tests/test_decomp.py trunk/scipy/linalg/tests/test_iterative.py trunk/scipy/linalg/tests/test_lapack.py trunk/scipy/linalg/tests/test_matfuncs.py trunk/scipy/linsolve/umfpack/tests/test_umfpack.py trunk/scipy/maxentropy/tests/test_maxentropy.py trunk/scipy/ndimage/tests/test_ndimage.py trunk/scipy/odr/tests/test_odr.py trunk/scipy/optimize/tests/test_cobyla.py trunk/scipy/optimize/tests/test_nonlin.py trunk/scipy/optimize/tests/test_optimize.py trunk/scipy/sandbox/arpack/tests/test_arpack.py trunk/scipy/sandbox/arpack/tests/test_speigs.py trunk/scipy/sandbox/cdavid/tests/test_autocorr.py trunk/scipy/sandbox/cdavid/tests/test_lpc.py trunk/scipy/sandbox/cdavid/tests/test_segmentaxis.py trunk/scipy/sandbox/dhuard/test_histogram.py trunk/scipy/sandbox/dhuard/test_stats.py trunk/scipy/sandbox/exmplpackage/tests/test_foo.py trunk/scipy/sandbox/exmplpackage/yyy/tests/test_yyy.py trunk/scipy/sandbox/fdfpack/tests/test_fdf.py trunk/scipy/sandbox/maskedarray/alternative_versions/test_mrecarray.py trunk/scipy/sandbox/maskedarray/tests/test_core.py trunk/scipy/sandbox/maskedarray/tests/test_extras.py trunk/scipy/sandbox/maskedarray/tests/test_morestats.py trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py trunk/scipy/sandbox/maskedarray/tests/test_mstats.py trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py trunk/scipy/sandbox/montecarlo/tests/test_dictsampler.py trunk/scipy/sandbox/montecarlo/tests/test_intsampler.py trunk/scipy/sandbox/multigrid/tests/test_adaptive.py trunk/scipy/sandbox/multigrid/tests/test_coarsen.py trunk/scipy/sandbox/multigrid/tests/test_relaxation.py trunk/scipy/sandbox/multigrid/tests/test_utils.py trunk/scipy/sandbox/numexpr/tests/test_numexpr.py trunk/scipy/sandbox/pyem/tests/test_densities.py trunk/scipy/sandbox/pyem/tests/test_examples.py trunk/scipy/sandbox/pyem/tests/test_gauss_mix.py trunk/scipy/sandbox/pyloess/tests/test_mpyloess.py trunk/scipy/sandbox/pyloess/tests/test_pyloess.py trunk/scipy/sandbox/rbf/tests/test_rbf.py trunk/scipy/sandbox/spline/tests/test_fitpack.py trunk/scipy/sandbox/spline/tests/test_spline.py trunk/scipy/sandbox/svm/tests/test_classification.py trunk/scipy/sandbox/svm/tests/test_dataset.py trunk/scipy/sandbox/svm/tests/test_kernel.py trunk/scipy/sandbox/svm/tests/test_libsvm.py trunk/scipy/sandbox/svm/tests/test_oneclass.py trunk/scipy/sandbox/svm/tests/test_regression.py trunk/scipy/sandbox/svm/tests/test_speed.py trunk/scipy/sandbox/timeseries/lib/tests/test_interpolate.py trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py trunk/scipy/sandbox/timeseries/tests/test_dates.py trunk/scipy/sandbox/timeseries/tests/test_extras.py trunk/scipy/sandbox/timeseries/tests/test_timeseries.py trunk/scipy/sandbox/timeseries/tests/test_trecords.py trunk/scipy/signal/tests/test_signaltools.py trunk/scipy/signal/tests/test_wavelets.py trunk/scipy/sparse/tests/test_sparse.py trunk/scipy/special/tests/test_basic.py trunk/scipy/special/tests/test_spfun_stats.py trunk/scipy/stats/models/tests/test_bspline.py trunk/scipy/stats/models/tests/test_formula.py trunk/scipy/stats/models/tests/test_glm.py trunk/scipy/stats/models/tests/test_regression.py trunk/scipy/stats/models/tests/test_rlm.py trunk/scipy/stats/models/tests/test_utils.py trunk/scipy/stats/tests/test_distributions.py trunk/scipy/stats/tests/test_morestats.py trunk/scipy/stats/tests/test_stats.py trunk/scipy/weave/tests/test_ast_tools.py trunk/scipy/weave/tests/test_blitz_tools.py trunk/scipy/weave/tests/test_build_tools.py trunk/scipy/weave/tests/test_c_spec.py trunk/scipy/weave/tests/test_catalog.py trunk/scipy/weave/tests/test_ext_tools.py trunk/scipy/weave/tests/test_inline_tools.py trunk/scipy/weave/tests/test_numpy_scalar_spec.py trunk/scipy/weave/tests/test_scxx_dict.py trunk/scipy/weave/tests/test_scxx_object.py trunk/scipy/weave/tests/test_size_check.py trunk/scipy/weave/tests/test_slice_handler.py trunk/scipy/weave/tests/test_standard_array_spec.py trunk/scipy/weave/tests/test_wx_spec.py Log: Rename test classes to CapWords. Modified: trunk/scipy/cluster/tests/test_vq.py =================================================================== --- trunk/scipy/cluster/tests/test_vq.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/cluster/tests/test_vq.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -43,7 +43,7 @@ LABEL1 = N.array([0, 1, 2, 2, 2, 2, 1, 2, 1, 1, 1]) -class test_vq(NumpyTestCase): +class TestVq(NumpyTestCase): def check_py_vq(self, level=1): initc = N.concatenate(([[X[0]], [X[1]], [X[2]]])) code = initc.copy() @@ -89,7 +89,7 @@ else: print "== not testing C imp of vq (rank 1) ==" -class test_kmean(NumpyTestCase): +class TestKMean(NumpyTestCase): def check_kmeans_simple(self, level=1): initc = N.concatenate(([[X[0]], [X[1]], [X[2]]])) code = initc.copy() Modified: trunk/scipy/fftpack/tests/test_basic.py =================================================================== --- trunk/scipy/fftpack/tests/test_basic.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/fftpack/tests/test_basic.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -90,7 +90,7 @@ x1[0] = x[0] return direct_idft(x1).real -class test_fft(NumpyTestCase): +class TestFft(NumpyTestCase): def check_definition(self): x = [1,2,3,4+1j,1,2,3,4+2j] @@ -162,7 +162,7 @@ print ' (secs for %s calls)' % (repeat) sys.stdout.flush() -class test_ifft(NumpyTestCase): +class TestIfft(NumpyTestCase): def check_definition(self): x = [1,2,3,4+1j,1,2,3,4+2j] @@ -237,7 +237,7 @@ print ' (secs for %s calls)' % (repeat) sys.stdout.flush() -class test_rfft(NumpyTestCase): +class TestRfft(NumpyTestCase): def check_definition(self): x = [1,2,3,4,1,2,3,4] @@ -292,7 +292,7 @@ print ' (secs for %s calls)' % (repeat) sys.stdout.flush() -class test_irfft(NumpyTestCase): +class TestIrfft(NumpyTestCase): def check_definition(self): x = [1,2,3,4,1,2,3,4] @@ -368,7 +368,7 @@ sys.stdout.flush() -class test_fftn(NumpyTestCase): +class TestFftn(NumpyTestCase): def check_definition(self): x = [[1,2,3],[4,5,6],[7,8,9]] @@ -529,7 +529,7 @@ sys.stdout.flush() -class test_ifftn(NumpyTestCase): +class TestIfftn(NumpyTestCase): def check_definition(self): x = [[1,2,3],[4,5,6],[7,8,9]] Modified: trunk/scipy/fftpack/tests/test_helper.py =================================================================== --- trunk/scipy/fftpack/tests/test_helper.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/fftpack/tests/test_helper.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -22,7 +22,7 @@ def random(size): return rand(*size) -class test_fftshift(NumpyTestCase): +class TestFFTShift(NumpyTestCase): def check_definition(self): x = [0,1,2,3,4,-4,-3,-2,-1] @@ -39,7 +39,7 @@ x = random((n,)) assert_array_almost_equal(ifftshift(fftshift(x)),x) -class test_fftfreq(NumpyTestCase): +class TestFFTFreq(NumpyTestCase): def check_definition(self): x = [0,1,2,3,4,-4,-3,-2,-1] @@ -49,7 +49,7 @@ assert_array_almost_equal(10*fftfreq(10),x) assert_array_almost_equal(10*pi*fftfreq(10,pi),x) -class test_rfftfreq(NumpyTestCase): +class TestRFFTFreq(NumpyTestCase): def check_definition(self): x = [0,1,1,2,2,3,3,4,4] Modified: trunk/scipy/fftpack/tests/test_pseudo_diffs.py =================================================================== --- trunk/scipy/fftpack/tests/test_pseudo_diffs.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/fftpack/tests/test_pseudo_diffs.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -77,7 +77,7 @@ return ifft(fft(x)*exp(k*a)).real -class test_diff(NumpyTestCase): +class TestDiff(NumpyTestCase): def check_definition(self): for n in [16,17,64,127,32]: @@ -216,7 +216,7 @@ print ' (secs for %s calls)' % (repeat) -class test_tilbert(NumpyTestCase): +class TestTilbert(NumpyTestCase): def check_definition(self): for h in [0.1,0.5,1,5.5,10]: @@ -277,7 +277,7 @@ sys.stdout.flush() print ' (secs for %s calls)' % (repeat) -class test_itilbert(NumpyTestCase): +class TestITilbert(NumpyTestCase): def check_definition(self): for h in [0.1,0.5,1,5.5,10]: @@ -291,7 +291,7 @@ assert_array_almost_equal(itilbert(sin(2*x),h), direct_itilbert(sin(2*x),h)) -class test_hilbert(NumpyTestCase): +class TestHilbert(NumpyTestCase): def check_definition(self): for n in [16,17,64,127]: @@ -360,7 +360,7 @@ sys.stdout.flush() print ' (secs for %s calls)' % (repeat) -class test_ihilbert(NumpyTestCase): +class TestIHilbert(NumpyTestCase): def check_definition(self): for n in [16,17,64,127]: @@ -381,7 +381,7 @@ y2 = itilbert(f,h=10) assert_array_almost_equal (y,y2) -class test_shift(NumpyTestCase): +class TestShift(NumpyTestCase): def check_definition(self): for n in [18,17,64,127,32,2048,256]: Modified: trunk/scipy/integrate/tests/test_integrate.py =================================================================== --- trunk/scipy/integrate/tests/test_integrate.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/integrate/tests/test_integrate.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -14,7 +14,7 @@ from scipy.integrate import odeint restore_path() -class test_odeint(NumpyTestCase): +class TestODEInt(NumpyTestCase): """ Test odeint: free vibration of a simple oscillator m \ddot{u} + k u = 0, u(0) = u_0 \dot{u}(0) \dot{u}_0 Modified: trunk/scipy/integrate/tests/test_quadpack.py =================================================================== --- trunk/scipy/integrate/tests/test_quadpack.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/integrate/tests/test_quadpack.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -10,7 +10,7 @@ if errTol is not None: assert err < errTol, (err, errTol) -class test_quad(NumpyTestCase): +class TestQuad(NumpyTestCase): def check_typical(self): # 1) Typical function with two extra arguments: def myfunc(x,n,z): # Bessel function integrand Modified: trunk/scipy/integrate/tests/test_quadrature.py =================================================================== --- trunk/scipy/integrate/tests/test_quadrature.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/integrate/tests/test_quadrature.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -7,7 +7,7 @@ from scipy.integrate import quadrature, romberg, romb restore_path() -class test_quadrature(NumpyTestCase): +class TestQuadrature(NumpyTestCase): def quad(self, x, a, b, args): raise NotImplementedError Modified: trunk/scipy/interpolate/tests/test_fitpack.py =================================================================== --- trunk/scipy/interpolate/tests/test_fitpack.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/interpolate/tests/test_fitpack.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -22,7 +22,7 @@ RectBivariateSpline restore_path() -class test_UnivariateSpline(NumpyTestCase): +class TestUnivariateSpline(NumpyTestCase): def check_linear_constant(self): x = [1,2,3] y = [3,3,3] @@ -41,7 +41,7 @@ assert_almost_equal(lut.get_residual(),0.0) assert_array_almost_equal(lut([1,1.5,2]),[0,1,2]) -class test_LSQBivariateSpline(NumpyTestCase): +class TestLSQBivariateSpline(NumpyTestCase): def check_linear_constant(self): x = [1,1,1,2,2,2,3,3,3] y = [1,2,3,1,2,3,1,2,3] @@ -54,7 +54,7 @@ #print lut.get_coeffs() #print lut.get_residual() -class test_SmoothBivariateSpline(NumpyTestCase): +class TestSmoothBivariateSpline(NumpyTestCase): def check_linear_constant(self): x = [1,1,1,2,2,2,3,3,3] y = [1,2,3,1,2,3,1,2,3] @@ -75,7 +75,7 @@ assert_almost_equal(lut.get_residual(),0.0) assert_array_almost_equal(lut([1,1.5,2],[1,1.5]),[[0,0],[1,1],[2,2]]) -class test_RectBivariateSpline(NumpyTestCase): +class TestRectBivariateSpline(NumpyTestCase): def check_defaults(self): x = array([1,2,3,4,5]) y = array([1,2,3,4,5]) Modified: trunk/scipy/interpolate/tests/test_interpolate.py =================================================================== --- trunk/scipy/interpolate/tests/test_interpolate.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/interpolate/tests/test_interpolate.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -7,18 +7,18 @@ restore_path() -class test_interp2d(NumpyTestCase): +class TestInterp2D(NumpyTestCase): def test_interp2d(self): y, x = mgrid[0:pi:20j, 0:pi:21j] z = sin(x+y) I = interp2d(x, y, z) assert_almost_equal(I(1.0, 1.0), sin(2.0), decimal=2) - v,u = ogrid[0:pi:24j, 0:pi:25j] + v,u = ogrid[0:pi:24j, 0:pi:25j] assert_almost_equal(I(u.ravel(), v.ravel()), sin(v+u), decimal=2) -class test_interp1d(NumpyTestCase): +class TestInterp1D(NumpyTestCase): def setUp(self): self.x10 = np.arange(10.) Modified: trunk/scipy/io/tests/test_array_import.py =================================================================== --- trunk/scipy/io/tests/test_array_import.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/io/tests/test_array_import.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -16,7 +16,7 @@ import numpy.oldnumeric as N import tempfile -class test_numpyio(NumpyTestCase): +class TestNumpyio(NumpyTestCase): def check_basic(self): # Generate some data a = 255*rand(20) @@ -34,7 +34,7 @@ assert(N.product(a.astype(N.Int16) == b,axis=0)) os.remove(fname) -class test_read_array(NumpyTestCase): +class TestReadArray(NumpyTestCase): def check_complex(self): a = rand(13,4) + 1j*rand(13,4) fname = tempfile.mktemp('.dat') Modified: trunk/scipy/io/tests/test_mio.py =================================================================== --- trunk/scipy/io/tests/test_mio.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/io/tests/test_mio.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -23,9 +23,9 @@ test_data_path = os.path.join(os.path.dirname(__file__), './data') -class test_mio_array(NumpyTestCase): +class TestMIOArray(NumpyTestCase): def __init__(self, *args, **kwargs): - super(test_mio_array, self).__init__(*args, **kwargs) + super(TestMIOArray, self).__init__(*args, **kwargs) def _check_level(self, label, expected, actual): """ Check one level of a potentially nested object / list """ Modified: trunk/scipy/io/tests/test_mmio.py =================================================================== --- trunk/scipy/io/tests/test_mmio.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/io/tests/test_mmio.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ import scipy restore_path() -class test_mmio_array(NumpyTestCase): +class TestMMIOArray(NumpyTestCase): def check_simple(self): a = [[1,2],[3,4]] @@ -136,7 +136,7 @@ 5 5 1.200e+01 ''' -class test_mmio_coordinate(NumpyTestCase): +class TestMMIOCoordinate(NumpyTestCase): def check_simple_todense(self): fn = mktemp() Modified: trunk/scipy/io/tests/test_npfile.py =================================================================== --- trunk/scipy/io/tests/test_npfile.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/io/tests/test_npfile.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from io.npfile import npfile, sys_endian_code restore_path() -class test_npfile(NumpyTestCase): +class TestNpFile(NumpyTestCase): def test_init(self): fd, fname = mkstemp() Modified: trunk/scipy/io/tests/test_recaster.py =================================================================== --- trunk/scipy/io/tests/test_recaster.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/io/tests/test_recaster.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -10,7 +10,7 @@ except: pass -class test_recaster(NumpyTestCase): +class TestRecaster(NumpyTestCase): def test_init(self): # Setting sctype_list Modified: trunk/scipy/lib/blas/tests/test_blas.py =================================================================== --- trunk/scipy/lib/blas/tests/test_blas.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/lib/blas/tests/test_blas.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -24,7 +24,7 @@ from blas import get_blas_funcs restore_path() -class test_cblas1_simple(NumpyTestCase): +class TestCBLAS1Simple(NumpyTestCase): def check_axpy(self): for p in 'sd': @@ -36,7 +36,7 @@ if f is None: continue assert_array_almost_equal(f([1,2j,3],[2,-1,3],a=5),[7,10j-1,18]) -class test_fblas1_simple(NumpyTestCase): +class TestFBLAS1Simple(NumpyTestCase): def check_axpy(self): for p in 'sd': @@ -122,7 +122,7 @@ assert_equal(f([-5,4+3j,6]),1) #XXX: need tests for rot,rotm,rotg,rotmg -class test_fblas2_simple(NumpyTestCase): +class TestFBLAS2Simple(NumpyTestCase): def check_gemv(self): for p in 'sd': @@ -170,7 +170,7 @@ 2j, 3j],[3j,4j]),[[6,8],[12,16],[18,24]]) -class test_fblas3_simple(NumpyTestCase): +class TestFBLAS3Simple(NumpyTestCase): def check_gemm(self): for p in 'sd': @@ -195,7 +195,7 @@ assert_array_almost_equal(f(1,[[1,2]],[[3],[4]]),[[11]]) assert_array_almost_equal(f(1,[[1,2],[1,2]],[[3],[4]]),[[11],[11]]) -class test_blas(NumpyTestCase): +class TestBLAS(NumpyTestCase): def check_blas(self): a = array([[1,1,1]]) Modified: trunk/scipy/linalg/tests/test_basic.py =================================================================== --- trunk/scipy/linalg/tests/test_basic.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linalg/tests/test_basic.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -39,7 +39,7 @@ data = add.outer(data,data) return data -class test_solve_banded(NumpyTestCase): +class TestSolveBanded(NumpyTestCase): def check_simple(self): @@ -54,7 +54,7 @@ x = solve_banded((l,u),ab,b) assert_array_almost_equal(numpy.dot(a,x),b) -class test_solve(NumpyTestCase): +class TestSolve(NumpyTestCase): def check_20Feb04_bug(self): a = [[1,1],[1.0,0]] # ok @@ -193,7 +193,7 @@ print ' (secs for %s calls)' % (repeat) -class test_inv(NumpyTestCase): +class TestInv(NumpyTestCase): def check_simple(self): a = [[1,2],[3,4]] @@ -265,7 +265,7 @@ print ' (secs for %s calls)' % (repeat) -class test_det(NumpyTestCase): +class TestDet(NumpyTestCase): def check_simple(self): a = [[1,2],[3,4]] @@ -340,7 +340,7 @@ b1 = dot(at, b) return solve(a1, b1) -class test_lstsq(NumpyTestCase): +class TestLstsq(NumpyTestCase): def check_random_overdet_large(self): #bug report: Nils Wagner n = 200 @@ -510,7 +510,7 @@ y = hankel([1,2,3],[3,4,5]) assert_array_equal(y,[[1,2,3],[2,3,4],[3,4,5]]) -class test_pinv(NumpyTestCase): +class TestPinv(NumpyTestCase): def check_simple(self): a=array([[1,2,3],[4,5,6.],[7,8,10]]) Modified: trunk/scipy/linalg/tests/test_blas.py =================================================================== --- trunk/scipy/linalg/tests/test_blas.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linalg/tests/test_blas.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -24,7 +24,7 @@ from linalg import cblas restore_path() -class test_cblas1_simple(NumpyTestCase): +class TestCBLAS1Simple(NumpyTestCase): def check_axpy(self): for p in 'sd': @@ -36,7 +36,7 @@ if f is None: continue assert_array_almost_equal(f(5,[1,2j,3],[2,-1,3]),[7,10j-1,18]) -class test_fblas1_simple(NumpyTestCase): +class TestFBLAS1Simple(NumpyTestCase): def check_axpy(self): for p in 'sd': @@ -128,7 +128,7 @@ assert_equal(f([-5,4+3j,6]),1) #XXX: need tests for rot,rotm,rotg,rotmg -class test_fblas2_simple(NumpyTestCase): +class TestFBLAS2Simple(NumpyTestCase): def check_gemv(self): for p in 'sd': @@ -176,7 +176,7 @@ 2j, 3j],[3j,4j]),[[6,8],[12,16],[18,24]]) -class test_fblas3_simple(NumpyTestCase): +class TestFBLAS3Simple(NumpyTestCase): def check_gemm(self): for p in 'sd': @@ -190,7 +190,7 @@ assert_array_almost_equal(f(3j,[3-4j],[-4]),[[-48-36j]]) assert_array_almost_equal(f(3j,[3-4j],[-4],3,[5j]),[-48-21j]) -class test_blas(NumpyTestCase): +class TestBLAS(NumpyTestCase): def check_fblas(self): if hasattr(fblas,'empty_module'): Modified: trunk/scipy/linalg/tests/test_decomp.py =================================================================== --- trunk/scipy/linalg/tests/test_decomp.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linalg/tests/test_decomp.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -31,7 +31,7 @@ def random(size): return rand(*size) -class test_eigvals(NumpyTestCase): +class TestEigVals(NumpyTestCase): def check_simple(self): a = [[1,2,3],[1,2,3],[2,5,6]] @@ -77,7 +77,7 @@ print ' (secs for %s calls)' % (repeat) -class test_eig(NumpyTestCase): +class TestEig(NumpyTestCase): def check_simple(self): a = [[1,2,3],[1,2,3],[2,5,6]] @@ -151,7 +151,7 @@ if all(isfinite(res[:, i])): assert_array_almost_equal(res[:, i], 0) -class test_eig_banded(NumpyTestCase): +class TestEigBanded(NumpyTestCase): def __init__(self, *args): NumpyTestCase.__init__(self, *args) @@ -436,7 +436,7 @@ -class test_lu(NumpyTestCase): +class TestLU(NumpyTestCase): def __init__(self, *args, **kw): NumpyTestCase.__init__(self, *args, **kw) @@ -519,7 +519,7 @@ self.med = self.vrect.astype(float32) self.cmed = self.vrect.astype(complex64) -class test_lu_solve(NumpyTestCase): +class TestLuSolve(NumpyTestCase): def check_lu(self): a = random((10,10)) b = random((10,)) @@ -531,7 +531,7 @@ assert_array_equal(x1,x2) -class test_svd(NumpyTestCase): +class TestSvd(NumpyTestCase): def check_simple(self): a = [[1,2,3],[1,20,3],[2,5,6]] @@ -604,7 +604,7 @@ for i in range(len(s)): sigma[i,i] = s[i] assert_array_almost_equal(dot(dot(u,sigma),vh),a) -class test_svdvals(NumpyTestCase): +class TestSVDVals(NumpyTestCase): def check_simple(self): a = [[1,2,3],[1,2,3],[2,5,6]] @@ -642,12 +642,12 @@ assert len(s)==2 assert s[0]>=s[1] -class test_diagsvd(NumpyTestCase): +class TestDiagsvd(NumpyTestCase): def check_simple(self): assert_array_almost_equal(diagsvd([1,0,0],3,3),[[1,0,0],[0,0,0],[0,0,0]]) -class test_cholesky(NumpyTestCase): +class TestCholesky(NumpyTestCase): def check_simple(self): a = [[8,2,3],[2,9,3],[3,3,6]] @@ -696,7 +696,7 @@ assert_array_almost_equal(cholesky(a,lower=1),c) -class test_qr(NumpyTestCase): +class TestQr(NumpyTestCase): def check_simple(self): a = [[8,2,3],[2,9,3],[5,3,6]] @@ -779,7 +779,7 @@ assert_array_almost_equal(dot(conj(transpose(q)),q),identity(n)) assert_array_almost_equal(dot(q,r),a) -class test_rq(NumpyTestCase): +class TestRq(NumpyTestCase): def check_simple(self): a = [[8,2,3],[2,9,3],[5,3,6]] @@ -844,7 +844,7 @@ transp = transpose any = sometrue -class test_schur(NumpyTestCase): +class TestSchur(NumpyTestCase): def check_simple(self): a = [[8,12,3],[2,9,3],[10,3,6]] @@ -856,7 +856,7 @@ tc2,zc2 = rsf2csf(tc,zc) assert_array_almost_equal(dot(dot(zc2,tc2),transp(conj(zc2))),a) -class test_hessenberg(NumpyTestCase): +class TestHessenberg(NumpyTestCase): def check_simple(self): a = [[-149, -50,-154], @@ -905,7 +905,7 @@ -class test_datanotshared(NumpyTestCase): +class TestDatanotshared(NumpyTestCase): def check_datanotshared(self): from scipy.linalg.decomp import _datanotshared Modified: trunk/scipy/linalg/tests/test_iterative.py =================================================================== --- trunk/scipy/linalg/tests/test_iterative.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linalg/tests/test_iterative.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -29,7 +29,7 @@ res = b-dot(A,x) #print "||A.x - b|| = " + str(norm(dot(A,x)-b)) -class test_iterative_solvers(NumpyTestCase): +class TestIterativeSolvers(NumpyTestCase): def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) self.setUp() Modified: trunk/scipy/linalg/tests/test_lapack.py =================================================================== --- trunk/scipy/linalg/tests/test_lapack.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linalg/tests/test_lapack.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -20,7 +20,7 @@ from linalg import clapack restore_path() -class test_flapack_simple(NumpyTestCase): +class TestFlapackSimple(NumpyTestCase): def check_gebal(self): a = [[1,2,3],[4,5,6],[7,8,9]] @@ -52,7 +52,7 @@ ht,tau,info = f(a) assert not info,`info` -class test_lapack(NumpyTestCase): +class TestLapack(NumpyTestCase): def check_flapack(self): if hasattr(flapack,'empty_module'): Modified: trunk/scipy/linalg/tests/test_matfuncs.py =================================================================== --- trunk/scipy/linalg/tests/test_matfuncs.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linalg/tests/test_matfuncs.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -25,7 +25,7 @@ from linalg import signm,logm,funm, sqrtm, expm, expm2, expm3 restore_path() -class test_signm(NumpyTestCase): +class TestSignM(NumpyTestCase): def check_nils(self): a = array([[ 29.2, -24.2, 69.5, 49.8, 7. ], @@ -67,7 +67,7 @@ r = signm(a) #XXX: what would be the correct result? -class test_logm(NumpyTestCase): +class TestLogM(NumpyTestCase): def check_nils(self): a = array([[ -2., 25., 0., 0., 0., 0., 0.], @@ -81,7 +81,7 @@ logm(m) -class test_sqrtm(NumpyTestCase): +class TestSqrtM(NumpyTestCase): def check_bad(self): # See http://www.maths.man.ac.uk/~nareports/narep336.ps.gz e = 2**-5 @@ -98,7 +98,7 @@ esa = sqrtm(a) assert_array_almost_equal(dot(esa,esa),a) -class test_expm(NumpyTestCase): +class TestExpM(NumpyTestCase): def check_zero(self): a = array([[0.,0],[0,0]]) assert_array_almost_equal(expm(a),[[1,0],[0,1]]) Modified: trunk/scipy/linsolve/umfpack/tests/test_umfpack.py =================================================================== --- trunk/scipy/linsolve/umfpack/tests/test_umfpack.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/linsolve/umfpack/tests/test_umfpack.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -18,7 +18,7 @@ restore_path() -class test_solvers(NumpyTestCase): +class TestSolvers(NumpyTestCase): """Tests inverting a sparse linear system""" def check_solve_complex_without_umfpack(self): @@ -104,7 +104,7 @@ -class test_factorization(NumpyTestCase): +class TestFactorization(NumpyTestCase): """Tests factorizing a sparse linear system""" def check_complex_lu(self): Modified: trunk/scipy/maxentropy/tests/test_maxentropy.py =================================================================== --- trunk/scipy/maxentropy/tests/test_maxentropy.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/maxentropy/tests/test_maxentropy.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -16,7 +16,7 @@ import unittest -class test_maxentropy(NumpyTestCase): +class TestMaxentropy(NumpyTestCase): """Test whether logsumexp() function correctly handles large inputs. """ Modified: trunk/scipy/ndimage/tests/test_ndimage.py =================================================================== --- trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -66,7 +66,7 @@ return math.sqrt(t) -class test_ndimage(NumpyTestCase): +class TestNdimage(NumpyTestCase): def setUp(self): # list of numarray data types Modified: trunk/scipy/odr/tests/test_odr.py =================================================================== --- trunk/scipy/odr/tests/test_odr.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/odr/tests/test_odr.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from scipy.odr import Data, Model, ODR, RealData, odr_stop -class test_odr(NumpyTestCase): +class TestODR(NumpyTestCase): # Explicit Example Modified: trunk/scipy/optimize/tests/test_cobyla.py =================================================================== --- trunk/scipy/optimize/tests/test_cobyla.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/optimize/tests/test_cobyla.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -6,7 +6,7 @@ restore_path() import math -class test_cobyla(NumpyTestCase): +class TestCobyla(NumpyTestCase): def check_simple(self, level=1): function = lambda x: x[0]**2 + abs(x[1])**3 Modified: trunk/scipy/optimize/tests/test_nonlin.py =================================================================== --- trunk/scipy/optimize/tests/test_nonlin.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/optimize/tests/test_nonlin.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -25,7 +25,7 @@ return tuple(f.flat) -class test_nonlin(NumpyTestCase): +class TestNonlin(NumpyTestCase): """ Test case for a simple constrained entropy maximization problem (the machine translation example of Berger et al in Computational Linguistics, vol 22, num 1, pp 39--72, 1996.) Modified: trunk/scipy/optimize/tests/test_optimize.py =================================================================== --- trunk/scipy/optimize/tests/test_optimize.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/optimize/tests/test_optimize.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -13,7 +13,7 @@ from math import sin, cos, pow -class test_optimize(NumpyTestCase): +class TestOptimize(NumpyTestCase): """ Test case for a simple constrained entropy maximization problem (the machine translation example of Berger et al in Computational Linguistics, vol 22, num 1, pp 39--72, 1996.) @@ -143,7 +143,7 @@ -class test_tnc(NumpyTestCase): +class TestTnc(NumpyTestCase): """TNC non-linear optimization. These tests are taken from Prof. K. Schittkowski's test examples Modified: trunk/scipy/sandbox/arpack/tests/test_arpack.py =================================================================== --- trunk/scipy/sandbox/arpack/tests/test_arpack.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/arpack/tests/test_arpack.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -19,7 +19,7 @@ import numpy from scipy.linalg import eig,eigh,norm -class test_eigen_nonsymmetric(NumpyTestCase): +class TestEigenNonsymmetric(NumpyTestCase): def get_a1(self,typ): mat=numpy.array([[-2., -8., 1., 2., -5.], @@ -121,7 +121,7 @@ -class test_eigen_complex_nonsymmetric(NumpyTestCase): +class TestEigenComplexNonsymmetric(NumpyTestCase): def get_a1(self,typ): mat=numpy.array([[-2., -8., 1., 2., -5.], @@ -218,7 +218,7 @@ -class test_eigen_symmetric(NumpyTestCase): +class TestEigenSymmetric(NumpyTestCase): def get_a1(self,typ): mat_a1=numpy.array([[ 2., 0., 0., -1., 0., -1.], @@ -290,7 +290,7 @@ self.end_eigenvalues(typ,k) -class test_eigen_complex_symmetric(NumpyTestCase): +class TestEigenComplexSymmetric(NumpyTestCase): def get_a1(self,typ): mat_a1=numpy.array([[ 2., 0., 0., -1., 0., -1.], Modified: trunk/scipy/sandbox/arpack/tests/test_speigs.py =================================================================== --- trunk/scipy/sandbox/arpack/tests/test_speigs.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/arpack/tests/test_speigs.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ import numpy as N -class test_eigs(NumpyTestCase): +class TestEigs(NumpyTestCase): def test(self): maxn=15 # Dimension of square matrix to be solved # Use a PDP^-1 factorisation to construct matrix with known @@ -36,7 +36,7 @@ assert_array_almost_equal(calc_vecs, N.array(vecs)[:,0:nev], decimal=7) -# class test_geneigs(NumpyTestCase): +# class TestGeneigs(NumpyTestCase): # def test(self): # import pickle # import scipy.linsolve Modified: trunk/scipy/sandbox/cdavid/tests/test_autocorr.py =================================================================== --- trunk/scipy/sandbox/cdavid/tests/test_autocorr.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/cdavid/tests/test_autocorr.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -42,7 +42,7 @@ # This class tests the C functions directly. This is more a debugging tool # that a test case, as the tested functions are not part of the public API -class test_ctype_1d(NumpyTestCase): +class TestCType1D(NumpyTestCase): def check_contiguous_double(self): # double test xt = xc1 @@ -85,7 +85,7 @@ assert_array_equal(yt, yr) # Test autocorrelation for rank 1 arrays -class test_autocorr_1d(NumpyTestCase): +class TestAutoCorr1D(NumpyTestCase): def check_contiguous_double(self): # double test xt = xc1 @@ -131,7 +131,7 @@ # with rank 2 arrays. This will be used in the above test cases; # this function implements the expected behaviour of the public # autocorr function. -class test_autocorr_py(NumpyTestCase): +class TestAutoCorrPy(NumpyTestCase): def check_full(self): xt = xc axis = -1 @@ -179,7 +179,7 @@ assert_array_equal(tmp[center:center+1+lag], yt[:, i]) # Test autocorrelation for rank 2 arrays -class test_autocorr_2d(NumpyTestCase): +class TestAutoCorr2D(NumpyTestCase): def check_double_full(self): # C, axis 1 test xt = xc @@ -291,7 +291,7 @@ yr = autocorr_py(xt, lag, axis = axis) assert_array_equal(yt, yr) -class test_autocorr_fft(NumpyTestCase): +class TestAutoCorrFFT(NumpyTestCase): n = 5 d = 3 def check_nextpow2(self): @@ -347,7 +347,7 @@ if __name__ == "__main__": NumpyTest().run() -#class test_autocorr_2d(NumpyTestCase): +#class TestAutocorr2d(NumpyTestCase): # def check_double(self): # # C, axis 1 test # xt = xc Modified: trunk/scipy/sandbox/cdavid/tests/test_lpc.py =================================================================== --- trunk/scipy/sandbox/cdavid/tests/test_lpc.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/cdavid/tests/test_lpc.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -41,7 +41,7 @@ # This class uses lpc in 1 dimension and loop on the axis. Is tested against # a direct matrix inversion of the autocorrelation matrix (using matrix inverse # instead of levinson durbin) -class test_lpc_py(NumpyTestCase): +class TestLpcPy(NumpyTestCase): def check_float(self): # Axis -1 xt = xcf @@ -100,7 +100,7 @@ assert_array_almost_equal(tmp, a) -class test_lpc(NumpyTestCase): +class TestLpc(NumpyTestCase): def check_float(self): # Axis -1 xt = xcf Modified: trunk/scipy/sandbox/cdavid/tests/test_segmentaxis.py =================================================================== --- trunk/scipy/sandbox/cdavid/tests/test_segmentaxis.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/cdavid/tests/test_segmentaxis.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -14,7 +14,7 @@ # # import modules that are located in the same directory as this file. # restore_path() -class test_segment(NumpyTestCase): +class TestSegment(NumpyTestCase): def check_simple(self): assert_equal(segment_axis(N.arange(6),length=3,overlap=0), N.array([[0,1,2],[3,4,5]])) Modified: trunk/scipy/sandbox/dhuard/test_histogram.py =================================================================== --- trunk/scipy/sandbox/dhuard/test_histogram.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/dhuard/test_histogram.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -4,7 +4,7 @@ import numpy as np from numpy.random import rand -class test_histogram1d_functions(NumpyTestCase): +class TestHistogram1DFunctions(NumpyTestCase): def check_consistency(self): n = 100 r = rand(n)*12-1 @@ -15,7 +15,7 @@ assert_array_equal(a,b) assert_array_equal(c,b) -class test_histogram(NumpyTestCase): +class TestHistogram(NumpyTestCase): def check_simple(self): n=100 v=rand(n) Modified: trunk/scipy/sandbox/dhuard/test_stats.py =================================================================== --- trunk/scipy/sandbox/dhuard/test_stats.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/dhuard/test_stats.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -11,35 +11,35 @@ np.random.seed(2) r = np.random.randn(N) -class test_empiricalcdf(NumpyTestCase): - def check_hazen(self): - - f = stats.empiricalcdf(r) - assert_equal(len(f), len(r)) - assert_array_equal(np.argsort(r), np.argsort(f)) - assert_array_equal(np.sort(f), (np.arange(N)+.5)/N) - - def check_weibull(self): - f = stats.empiricalcdf(r, 'weibull') - assert_array_equal(np.sort(f), (np.arange(N)+1.)/(N+1.)) +class TestEmpiricalCDF(NumpyTestCase): + def check_hazen(self): - def check_california(self): - f = stats.empiricalcdf(r, 'california') - assert_array_equal(np.sort(f), (np.arange(N))/float(N)) - -class test_scoreatpercentile(NumpyTestCase): - def check_simple(self): - r = np.random.randn(1000) - s = stats.scoreatpercentile(r, [15.9,50,84.1]) - assert_array_almost_equal(s, [-1,0,1], 1) - -class test_percentileofscore(NumpyTestCase): - def check_simple(self): - r = np.random.randn(3000) - p = stats.percentileofscore(r, [-1,0,1]) - assert_array_almost_equal(p, [15.9, 50, 84.1], 0) - - - + f = stats.empiricalcdf(r) + assert_equal(len(f), len(r)) + assert_array_equal(np.argsort(r), np.argsort(f)) + assert_array_equal(np.sort(f), (np.arange(N)+.5)/N) + + def check_weibull(self): + f = stats.empiricalcdf(r, 'weibull') + assert_array_equal(np.sort(f), (np.arange(N)+1.)/(N+1.)) + + def check_california(self): + f = stats.empiricalcdf(r, 'california') + assert_array_equal(np.sort(f), (np.arange(N))/float(N)) + +class TestScoreAtPercentile(NumpyTestCase): + def check_simple(self): + r = np.random.randn(1000) + s = stats.scoreatpercentile(r, [15.9,50,84.1]) + assert_array_almost_equal(s, [-1,0,1], 1) + +class TestPercentileOfScore(NumpyTestCase): + def check_simple(self): + r = np.random.randn(3000) + p = stats.percentileofscore(r, [-1,0,1]) + assert_array_almost_equal(p, [15.9, 50, 84.1], 0) + + + if __name__ == '__main__': - NumpyTest().run() + NumpyTest().run() Modified: trunk/scipy/sandbox/exmplpackage/tests/test_foo.py =================================================================== --- trunk/scipy/sandbox/exmplpackage/tests/test_foo.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/exmplpackage/tests/test_foo.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -22,12 +22,12 @@ from exmplpackage.foo import * del sys.path[0] -class test_foo_bar(NumpyTestCase): +class TestFooBar(NumpyTestCase): def check_simple(self, level=1): assert exmplpackage_foo_bar()=='Hello from exmplpackage_foo_bar' -class test_foo_gun(NumpyTestCase): +class TestFooGun(NumpyTestCase): def check_simple(self, level=1): assert foo_gun()=='Hello from foo_gun' Modified: trunk/scipy/sandbox/exmplpackage/yyy/tests/test_yyy.py =================================================================== --- trunk/scipy/sandbox/exmplpackage/yyy/tests/test_yyy.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/exmplpackage/yyy/tests/test_yyy.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -7,7 +7,7 @@ from yyy import fun del sys.path[0] -class test_fun(NumpyTestCase): +class TestFun(NumpyTestCase): def check_simple(self, level=1): assert fun()=='Hello from yyy.fun' #... Modified: trunk/scipy/sandbox/fdfpack/tests/test_fdf.py =================================================================== --- trunk/scipy/sandbox/fdfpack/tests/test_fdf.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/fdfpack/tests/test_fdf.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -7,7 +7,7 @@ from numpy import arange, add, array,sin,cos,pi -class test_diff(NumpyTestCase): +class TestDiff(NumpyTestCase): def check_1(self): for n in [64,100,125,4000]: x = arange(n)*2*pi/n Modified: trunk/scipy/sandbox/maskedarray/alternative_versions/test_mrecarray.py =================================================================== --- trunk/scipy/sandbox/maskedarray/alternative_versions/test_mrecarray.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/alternative_versions/test_mrecarray.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -26,7 +26,7 @@ #.............................................................................. -class test_mrecarray(NumpyTestCase): +class TestMrecarray(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/maskedarray/tests/test_core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_core.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/tests/test_core.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -27,7 +27,7 @@ pi = numpy.pi #.............................................................................. -class test_ma(NumpyTestCase): +class TestMa(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -786,7 +786,7 @@ #............................................................................... -class test_ufuncs(NumpyTestCase): +class TestUfuncs(NumpyTestCase): "Test class for the application of ufuncs on MaskedArrays." def setUp(self): "Base data definition." @@ -848,7 +848,7 @@ #............................................................................... -class test_array_methods(NumpyTestCase): +class TestArrayMethods(NumpyTestCase): "Test class for miscellaneous MaskedArrays methods." def setUp(self): "Base data definition." Modified: trunk/scipy/sandbox/maskedarray/tests/test_extras.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_extras.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/tests/test_extras.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -23,7 +23,7 @@ import maskedarray.extras from maskedarray.extras import * -class test_average(NumpyTestCase): +class TestAverage(NumpyTestCase): "Several tests of average. Why so many ? Good point..." def check_testAverage1(self): "Test of average." @@ -98,7 +98,7 @@ a2dma = average(a2dm, axis=1) assert_equal(a2dma, [1.5, 4.0]) -class test_concatenator(NumpyTestCase): +class TestConcatenator(NumpyTestCase): "Tests for mr_, the equivalent of r_ for masked arrays." def check_1d(self): "Tests mr_ on 1D arrays." @@ -130,7 +130,7 @@ assert_array_equal(d[5:,:],b_2) assert_array_equal(d.mask, N.r_[m_1,m_2]) -class test_notmasked(NumpyTestCase): +class TestNotMasked(NumpyTestCase): "Tests notmasked_edges and notmasked_contiguous." def check_edges(self): "Tests unmasked_edges" @@ -172,7 +172,7 @@ assert_equal(tmp[2][-1], slice(7,7,None)) assert_equal(tmp[2][-2], slice(0,5,None)) -class test_2dfunctions(NumpyTestCase): +class Test2DFunctions(NumpyTestCase): "Tests 2D functions" def check_compress2d(self): "Tests compress2d" @@ -316,7 +316,7 @@ assert_equal(dx._data, N.r_[0,difx_d,0]) assert_equal(dx._mask, N.r_[1,0,0,0,0,1]) -class test_apply_along_axis(NumpyTestCase): +class TestApplyAlongAxis(NumpyTestCase): "Tests 2D functions" def check_3d(self): a = arange(12.).reshape(2,2,3) @@ -330,4 +330,4 @@ if __name__ == "__main__": NumpyTest().run() - \ No newline at end of file + Modified: trunk/scipy/sandbox/maskedarray/tests/test_morestats.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_morestats.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/tests/test_morestats.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -24,7 +24,7 @@ from maskedarray.testutils import * -class test_misc(NumpyTestCase): +class TestMisc(NumpyTestCase): # def __init__(self, *args, **kwargs): NumpyTestCase.__init__(self, *args, **kwargs) @@ -43,7 +43,7 @@ assert_equal(numpy.round(trimmed_mean_ci(data,0.2),1), [561.8, 630.6]) #.............................................................................. -class test_ranking(NumpyTestCase): +class TestRanking(NumpyTestCase): # def __init__(self, *args, **kwargs): NumpyTestCase.__init__(self, *args, **kwargs) @@ -63,7 +63,7 @@ assert_almost_equal(rank_data(x,axis=0),[[1,1,1,1,1],[2,2,2,2,2,]]) #.............................................................................. -class test_quantiles(NumpyTestCase): +class TestQuantiles(NumpyTestCase): # def __init__(self, *args, **kwargs): NumpyTestCase.__init__(self, *args, **kwargs) Modified: trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -30,7 +30,7 @@ fromarrays, fromtextfile, fromrecords, addfield #.............................................................................. -class test_mrecords(NumpyTestCase): +class TestMrecords(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/maskedarray/tests/test_mstats.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_mstats.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/tests/test_mstats.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -21,7 +21,7 @@ from maskedarray.mstats import * #.............................................................................. -class test_quantiles(NumpyTestCase): +class TestQuantiles(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -81,7 +81,7 @@ assert_almost_equal(mquantiles(b, axis=1), maskedarray.resize([24.9, 50., 75.1], (100,3))) -class test_median(NumpyTestCase): +class TestMedian(NumpyTestCase): def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -111,7 +111,7 @@ assert_equal(mmedian(x,0), [[12,10],[8,9],[16,17]]) #.............................................................................. -class test_trimming(NumpyTestCase): +class TestTrimming(NumpyTestCase): # def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -153,7 +153,7 @@ assert_equal(winsorized.mask, data.mask) #.............................................................................. -class test_misc(NumpyTestCase): +class TestMisc(NumpyTestCase): def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -69,7 +69,7 @@ -class test_subclassing(NumpyTestCase): +class TestSubclassing(NumpyTestCase): """Test suite for masked subclasses of ndarray.""" def check_data_subclassing(self): Modified: trunk/scipy/sandbox/montecarlo/tests/test_dictsampler.py =================================================================== --- trunk/scipy/sandbox/montecarlo/tests/test_dictsampler.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/montecarlo/tests/test_dictsampler.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -21,7 +21,7 @@ import unittest -class test_dictsampler(NumpyTestCase): +class TestDictSampler(NumpyTestCase): def check_simple(self): """ # Sample from this discrete distribution: Modified: trunk/scipy/sandbox/montecarlo/tests/test_intsampler.py =================================================================== --- trunk/scipy/sandbox/montecarlo/tests/test_intsampler.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/montecarlo/tests/test_intsampler.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -30,7 +30,7 @@ import unittest -class test_intsampler(NumpyTestCase): +class TestIntSampler(NumpyTestCase): def check_simple(self): # Sample from a Poisson distribution, P(lambda = 10.0) lam = 10.0 Modified: trunk/scipy/sandbox/multigrid/tests/test_adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_adaptive.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/multigrid/tests/test_adaptive.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ restore_path() -class test_fit_candidates(NumpyTestCase): +class TestFitCandidates(NumpyTestCase): def setUp(self): self.cases = [] Modified: trunk/scipy/sandbox/multigrid/tests/test_coarsen.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_coarsen.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/multigrid/tests/test_coarsen.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -79,7 +79,7 @@ return csr_matrix((Px,Pj,Pp)) -class test_sa_strong_connections(NumpyTestCase): +class TestSaStrongConnections(NumpyTestCase): def check_simple(self): N = 4 A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() @@ -120,7 +120,7 @@ assert_array_equal(S_result.todense(),S_expected.todense()) -class test_sa_constant_interpolation(NumpyTestCase): +class TestSaConstantInterpolation(NumpyTestCase): def check_random(self): numpy.random.seed(0) for N in [2,3,5,10]: Modified: trunk/scipy/sandbox/multigrid/tests/test_relaxation.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_relaxation.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/multigrid/tests/test_relaxation.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -12,7 +12,7 @@ restore_path() -class test_relaxation(NumpyTestCase): +class TestRelaxation(NumpyTestCase): def check_polynomial(self): N = 3 A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).T Modified: trunk/scipy/sandbox/multigrid/tests/test_utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_utils.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/multigrid/tests/test_utils.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -11,7 +11,7 @@ restore_path() -class test_utils(NumpyTestCase): +class TestUtils(NumpyTestCase): def check_infinity_norm(self): A = matrix([[-4]]) assert_equal(infinity_norm(csr_matrix(A)),4) Modified: trunk/scipy/sandbox/numexpr/tests/test_numexpr.py =================================================================== --- trunk/scipy/sandbox/numexpr/tests/test_numexpr.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/numexpr/tests/test_numexpr.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -6,7 +6,7 @@ from numexpr import E, numexpr, evaluate, disassemble restore_path() -class test_numexpr(NumpyTestCase): +class TestNumExpr(NumpyTestCase): def check_simple(self): ex = 2.0 * E.a + 3.0 * E.b * E.c func = numexpr(ex, signature=[('a', float), ('b', float), ('c', float)]) @@ -95,7 +95,7 @@ [('mul_fff', 'r0', 'r1[x]', 'r1[x]'), ('add_fff', 'r0', 'r0', 'c2[2.0]')]) -class test_evaluate(NumpyTestCase): +class TestEvaluate(NumpyTestCase): def check_simple(self): a = array([1., 2., 3.]) b = array([4., 5., 6.]) @@ -239,7 +239,7 @@ class Skip(Exception): pass -class test_expressions(NumpyTestCase): +class TestExpressions(NumpyTestCase): pass def generate_check_expressions(): Modified: trunk/scipy/sandbox/pyem/tests/test_densities.py =================================================================== --- trunk/scipy/sandbox/pyem/tests/test_densities.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/pyem/tests/test_densities.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -105,7 +105,7 @@ #===================== # Basic speed tests #===================== -class test_speed(NumpyTestCase): +class TestSpeed(NumpyTestCase): def __init__(self, *args, **kw): NumpyTestCase.__init__(self, *args, **kw) import sys @@ -234,7 +234,7 @@ self._generate_test_data_2d_full() self._test(level) -class test_gauss_ell(NumpyTestCase): +class TestGaussEll(NumpyTestCase): def test_dim(self): pyem.densities.gauss_ell([0, 1], [1, 2.], [0, 1]) try: Modified: trunk/scipy/sandbox/pyem/tests/test_examples.py =================================================================== --- trunk/scipy/sandbox/pyem/tests/test_examples.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/pyem/tests/test_examples.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -12,7 +12,7 @@ # # import modules that are located in the same directory as this file. # restore_path() -class test_examples(NumpyTestCase): +class TestExamples(NumpyTestCase): def test_ex1(self, level = 3): ex1() Modified: trunk/scipy/sandbox/pyem/tests/test_gauss_mix.py =================================================================== --- trunk/scipy/sandbox/pyem/tests/test_gauss_mix.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/pyem/tests/test_gauss_mix.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -13,7 +13,7 @@ from pyem.densities import multiple_gauss_den restore_path() -class test_BasicFunc(NumpyTestCase): +class TestBasicfunc(NumpyTestCase): """Check that basic functionalities work.""" def test_conf_ellip(self): """Only test whether the call succeed. To check wether the result is Modified: trunk/scipy/sandbox/pyloess/tests/test_mpyloess.py =================================================================== --- trunk/scipy/sandbox/pyloess/tests/test_mpyloess.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/pyloess/tests/test_mpyloess.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -34,7 +34,7 @@ #####--------------------------------------------------------------------------- #---- --- LOWESS --- #####--------------------------------------------------------------------------- -class test_lowess(NumpyTestCase): +class TestLowess(NumpyTestCase): "Test class for lowess." # def __init__(self, *args, **kwds): @@ -95,7 +95,7 @@ #####--------------------------------------------------------------------------- #---- --- STL --- #####--------------------------------------------------------------------------- -class test_stl(NumpyTestCase): +class TestStl(NumpyTestCase): "Tests STL." # def __init__(self, *args, **kwds): @@ -141,7 +141,7 @@ #---- --- LOESS --- #####--------------------------------------------------------------------------- -class test_loess2d(NumpyTestCase): +class TestLoess2D(NumpyTestCase): "Test class for lowess." # def __init__(self, *args, **kwds): @@ -292,7 +292,7 @@ #####--------------------------------------------------------------------------- #---- --- test 1D --- #####--------------------------------------------------------------------------- -class test_loess_gas(NumpyTestCase): +class TestLoessGas(NumpyTestCase): "Test class for lowess." # def __init__(self, *args, **kwds): Modified: trunk/scipy/sandbox/pyloess/tests/test_pyloess.py =================================================================== --- trunk/scipy/sandbox/pyloess/tests/test_pyloess.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/pyloess/tests/test_pyloess.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -32,7 +32,7 @@ #####--------------------------------------------------------------------------- #---- --- LOWESS --- #####--------------------------------------------------------------------------- -class test_lowess(NumpyTestCase): +class TestLowess(NumpyTestCase): "Test class for lowess." # def __init__(self, *args, **kwds): @@ -78,7 +78,7 @@ #####--------------------------------------------------------------------------- #---- --- STL --- #####--------------------------------------------------------------------------- -class test_stl(NumpyTestCase): +class TestStl(NumpyTestCase): "Tests STL." # def __init__(self, *args, **kwds): @@ -124,7 +124,7 @@ #---- --- LOESS --- #####--------------------------------------------------------------------------- -class test_loess2d(NumpyTestCase): +class TestLoess2d(NumpyTestCase): "Test class for lowess." # def __init__(self, *args, **kwds): @@ -273,7 +273,7 @@ #####--------------------------------------------------------------------------- #---- --- test 2D #####--------------------------------------------------------------------------- -class test_loess_gas(NumpyTestCase): +class TestLoessGas(NumpyTestCase): "Test class for lowess." # def __init__(self, *args, **kwds): Modified: trunk/scipy/sandbox/rbf/tests/test_rbf.py =================================================================== --- trunk/scipy/sandbox/rbf/tests/test_rbf.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/rbf/tests/test_rbf.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ from rbf.rbf import Rbf restore_path() -class test_Rbf1D(NumpyTestCase): +class TestRbf1d(NumpyTestCase): def check_multiquadrics(self): x = linspace(0,10,9) y = sin(x) @@ -17,7 +17,7 @@ yi = rbf(x) assert_array_almost_equal(y, yi) -class test_Rbf2D(NumpyTestCase): +class TestRbf2d(NumpyTestCase): def check_multiquadrics(self): x = random.rand(50,1)*4-2 y = random.rand(50,1)*4-2 @@ -27,7 +27,7 @@ zi.shape = x.shape assert_array_almost_equal(z, zi) -class test_Rbf3D(NumpyTestCase): +class TestRbf3d(NumpyTestCase): def check_multiquadrics(self): x = random.rand(50,1)*4-2 y = random.rand(50,1)*4-2 Modified: trunk/scipy/sandbox/spline/tests/test_fitpack.py =================================================================== --- trunk/scipy/sandbox/spline/tests/test_fitpack.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/spline/tests/test_fitpack.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -24,7 +24,7 @@ from dierckx_test_data import * restore_path() -class test_splrep_splev(NumpyTestCase): +class TestSplrepSplev(NumpyTestCase): def check_curfit_against_dierckx_smth(self): x,y = curfit_test['x'],curfit_test['y'] k,s = curfit_test_smth['k'],curfit_test_smth['s'] @@ -83,7 +83,7 @@ yy = asarray(splev(x,tck)) assert_array_almost_equal(yy,sp[i], decimal=3) -class test_splprep_splev(NumpyTestCase): +class TestSplprepSplev(NumpyTestCase): def check_parcur_against_dierckx(self): xa,xo = parcur_test['xa'], parcur_test['xo'] k,s = parcur_test['k'], parcur_test['s'] @@ -155,7 +155,7 @@ yy[1::2] = y[1,:-1] assert_array_almost_equal(yy,sp[i], decimal=3) -class test_splint_spalde(NumpyTestCase): +class TestSplintSpalde(NumpyTestCase): def check_splint_spalde(self): per = [0, 1, 0] N = [20, 20, 50] @@ -178,7 +178,7 @@ assert_almost_equal(1, ddr/f1(dx,d), decimal=2) d=d+1 -class test_splder(NumpyTestCase): +class TestSplder(NumpyTestCase): def check_splder(self): N = 50 a,b = 0,2*pi @@ -192,7 +192,7 @@ d2 = splev(dx,tck,der=1) assert_almost_equal(1, dr[1]/f1(dx,1.0), decimal=2) -class test_sproot(NumpyTestCase): +class TestSproot(NumpyTestCase): def check_sproot(self): a=0 b=15 @@ -204,7 +204,7 @@ ex = array([0.0, pi, 2.0*pi, 3.0*pi, 4.0*pi]) assert_array_almost_equal(sproot(tck),ex, decimal=3) -class test_bisplev_bisplrep(NumpyTestCase): +class TestBisplevBisplrep(NumpyTestCase): def test_bisplev_bisplrep(self): f=f2; kx=3; ky=3; xb=0; xe=2*pi yb=0; ye=2*pi; Nx=20; Ny=20; s=0 @@ -219,7 +219,7 @@ v2.shape=len(tt[0]),len(tt[1]) assert_almost_equal(0.0, norm2(ravel(v1-v2)), decimal=5) -class test_parcur(NumpyTestCase): +class TestParcur(NumpyTestCase): def check_parcur(self): f=f1; per=0; s=0; a=0; b=2*pi; N=[20,50] Modified: trunk/scipy/sandbox/spline/tests/test_spline.py =================================================================== --- trunk/scipy/sandbox/spline/tests/test_spline.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/spline/tests/test_spline.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -27,7 +27,7 @@ from dierckx_test_data import * restore_path() -class test_UnivariateSpline(NumpyTestCase): +class TestUnivariateSpline(NumpyTestCase): def check_linear_constant(self): x = [1,2,3] y = [3,3,3] @@ -103,7 +103,7 @@ ex = array([0.0, pi, 2.0*pi, 3.0*pi, 4.0*pi]) assert_array_almost_equal(uspl.roots(),ex, decimal=3) -class test_LSQUnivariateSpline(NumpyTestCase): +class TestLSQUnivariateSpline(NumpyTestCase): def check_curfit_against_dierckx(self): """ Test against results obtined from the pure fortran routines. @@ -121,7 +121,7 @@ assert_array_almost_equal(around(lsquspl(x),1), curfit_test_lsq['sp'][i]) -class test_LSQBivariateSpline(NumpyTestCase): +class TestLSQBivariateSpline(NumpyTestCase): def check_linear_constant(self): x = [1,1,1,2,2,2,3,3,3] y = [1,2,3,1,2,3,1,2,3] @@ -134,7 +134,7 @@ #print lut.get_coeffs() #print lut.get_residual() -class test_SmoothBivariateSpline(NumpyTestCase): +class TestSmoothBivariateSpline(NumpyTestCase): def check_linear_constant(self): x = [1,1,1,2,2,2,3,3,3] y = [1,2,3,1,2,3,1,2,3] @@ -155,7 +155,7 @@ assert_almost_equal(lut.get_residual(),0.0) assert_array_almost_equal(lut([1,1.5,2],[1,1.5]),[[0,0],[1,1],[2,2]]) -class test_RectBivariateSpline(NumpyTestCase): +class TestRectBivariateSpline(NumpyTestCase): def check_defaults(self): x = array([1,2,3,4,5]) y = array([1,2,3,4,5]) Modified: trunk/scipy/sandbox/svm/tests/test_classification.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_classification.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_classification.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ from svm.predict import * restore_path() -class test_classification(NumpyTestCase): +class TestClassification(NumpyTestCase): def check_basics(self): kernel = LinearKernel() # C-SVC Modified: trunk/scipy/sandbox/svm/tests/test_dataset.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_dataset.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_dataset.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from svm.libsvm import svm_node_dtype restore_path() -class test_dataset(NumpyTestCase): +class TestDataset(NumpyTestCase): def check_convert_dict(self): x = N.array([(-1,0.)], dtype=svm_node_dtype) assert_array_equal(convert_to_svm_node({}), x) @@ -73,7 +73,7 @@ for i, x in enumerate(dataset): assert_array_equal(data[i], x[1]['value'][:-1]) -class test_svm_node_dot(NumpyTestCase): +class TestSVMNodeDot(NumpyTestCase): def check_basics(self): kernel = LinearKernel() @@ -88,7 +88,7 @@ y = N.array([(3,2.),(-1,0.)], dtype=svm_node_dtype) self.assertAlmostEqual(svm_node_dot(x, y, kernel), 4.) -class test_precomputed_dataset(NumpyTestCase): +class TestPrecomputedDataset(NumpyTestCase): def check_precompute(self): degree, gamma, coef0 = 4, 3.0, 2.0 kernels = [ Modified: trunk/scipy/sandbox/svm/tests/test_kernel.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_kernel.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_kernel.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ def kernelfunc(x, y): return 8 * N.dot(x, y.T) -class test_kernel(NumpyTestCase): +class TestKernel(NumpyTestCase): def check_linear_kernel(self): kernel = LinearKernel() x = N.array([2.]) Modified: trunk/scipy/sandbox/svm/tests/test_libsvm.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_libsvm.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_libsvm.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -5,7 +5,7 @@ import svm.libsvm as libsvm restore_path() -class test_libsvm(NumpyTestCase): +class TestLibSVM(NumpyTestCase): def check_svm_node(self): node = libsvm.svm_node() node = N.empty((), dtype=libsvm.svm_node_dtype) Modified: trunk/scipy/sandbox/svm/tests/test_oneclass.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_oneclass.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_oneclass.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from svm.predict import * restore_path() -class test_oneclass(NumpyTestCase): +class TestOneclass(NumpyTestCase): def check_basics(self): ModelType = LibSvmOneClassModel kernel = LinearKernel() Modified: trunk/scipy/sandbox/svm/tests/test_regression.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_regression.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_regression.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from svm.regression import * restore_path() -class test_regression(NumpyTestCase): +class TestRegression(NumpyTestCase): def check_basics(self): Model = LibSvmEpsilonRegressionModel kernel = LinearKernel() Modified: trunk/scipy/sandbox/svm/tests/test_speed.py =================================================================== --- trunk/scipy/sandbox/svm/tests/test_speed.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/svm/tests/test_speed.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from svm.predict import * restore_path() -class test_classification_speed(NumpyTestCase): +class TestClassificationSpeed(NumpyTestCase): def check_large_test_dataset(self): x = N.random.randn(150, 3) Modified: trunk/scipy/sandbox/timeseries/lib/tests/test_interpolate.py =================================================================== --- trunk/scipy/sandbox/timeseries/lib/tests/test_interpolate.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/timeseries/lib/tests/test_interpolate.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -21,7 +21,7 @@ from timeseries.lib.interpolate import backward_fill, forward_fill, interp_masked1d -class test_funcs(NumpyTestCase): +class TestFuncs(NumpyTestCase): def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py =================================================================== --- trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -28,7 +28,7 @@ from timeseries.lib import moving_funcs as MF -class test_cmov_average(NumpyTestCase): +class TestCMovAverage(NumpyTestCase): def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -84,7 +84,7 @@ -class test_mov_funcs(NumpyTestCase): +class TestMovFuncs(NumpyTestCase): def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -143,4 +143,4 @@ #------------------------------------------------------------------------------ if __name__ == "__main__": - NumpyTest().run() \ No newline at end of file + NumpyTest().run() Modified: trunk/scipy/sandbox/timeseries/tests/test_dates.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_dates.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/timeseries/tests/test_dates.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -33,7 +33,7 @@ from timeseries.cseries import freq_dict -class test_creation(NumpyTestCase): +class TestCreation(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): @@ -140,7 +140,7 @@ assert_equal(date_array(n, n+2), d) print "finished test_shortcuts" -class test_date_properties(NumpyTestCase): +class TestDateProperties(NumpyTestCase): "Test properties such as year, month, day_of_week, etc...." def __init__(self, *args, **kwds): @@ -245,7 +245,7 @@ def noWrap(item): return item -class test_freq_conversion(NumpyTestCase): +class TestFreqConversion(NumpyTestCase): "Test frequency conversion of date objects" def __init__(self, *args, **kwds): @@ -833,7 +833,7 @@ assert_func(date_S_end_of_minute.asfreq('T'), date_S_to_T) -class test_methods(NumpyTestCase): +class TestMethods(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): Modified: trunk/scipy/sandbox/timeseries/tests/test_extras.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_extras.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/timeseries/tests/test_extras.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -23,7 +23,7 @@ from timeseries.extras import * #.............................................................................. -class test_misc(NumpyTestCase): +class TestMisc(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -33,7 +33,7 @@ assert_equal(leap, [0,0,0,0,1,1,0,0,0,1]) #.............................................................................. -class test_countmissing(NumpyTestCase): +class TestCountmissing(NumpyTestCase): # def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/timeseries/tests/test_timeseries.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_timeseries.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/timeseries/tests/test_timeseries.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -31,7 +31,7 @@ mask_period, align_series, align_with, fill_missing_dates, tsmasked, \ concatenate_series, stack, split -class test_creation(NumpyTestCase): +class TestCreation(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -117,7 +117,7 @@ assert_equal(series._mask,[0,0,1]) #............................................................................... -class test_arithmetics(NumpyTestCase): +class TestArithmetics(NumpyTestCase): "Some basic arithmetic tests" def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -215,7 +215,7 @@ #............................................................................... -class test_getitem(NumpyTestCase): +class TestGetitem(NumpyTestCase): "Some getitem tests" def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -320,7 +320,7 @@ assert_equal(series[:,:,0], series._data[:,:,0]) assert_equal(series[:,:,0]._dates, series._dates) -class test_functions(NumpyTestCase): +class TestFunctions(NumpyTestCase): "Some getitem tests" def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/timeseries/tests/test_trecords.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_trecords.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sandbox/timeseries/tests/test_trecords.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -33,7 +33,7 @@ #.............................................................................. -class test_mrecords(NumpyTestCase): +class TestMRecords(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) @@ -186,4 +186,4 @@ ############################################################################### #------------------------------------------------------------------------------ if __name__ == "__main__": - NumpyTest().run() \ No newline at end of file + NumpyTest().run() Modified: trunk/scipy/signal/tests/test_signaltools.py =================================================================== --- trunk/scipy/signal/tests/test_signaltools.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/signal/tests/test_signaltools.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -7,27 +7,27 @@ from numpy import array, arange -class test_convolve(NumpyTestCase): +class TestConvolve(NumpyTestCase): def check_basic(self): a = [3,4,5,6,5,4] b = [1,2,3] c = signal.convolve(a,b) assert_array_equal(c,array([3,10,22,28,32,32,23,12])) -class test_medfilt(NumpyTestCase): +class TestMedFilt(NumpyTestCase): def check_basic(self): f = [[3,4,5],[2,3,4],[1,2,5]] d = signal.medfilt(f) assert_array_equal(d, [[0,3,0],[2,3,3],[0,2,0]]) -class test_wiener(NumpyTestCase): +class TestWiener(NumpyTestCase): def check_basic(self): g = array([[5,6,4,3],[3,5,6,2],[2,3,5,6],[1,6,9,7]],'d') correct = array([[2.16374269,3.2222222222, 2.8888888889, 1.6666666667],[2.666666667, 4.33333333333, 4.44444444444, 2.8888888888],[2.222222222, 4.4444444444, 5.4444444444, 4.801066874837],[1.33333333333, 3.92735042735, 6.0712560386, 5.0404040404]]) h = signal.wiener(g) assert_array_almost_equal(h,correct,decimal=6) -class test_cspline1d_eval(NumpyTestCase): +class TestCSpline1DEval(NumpyTestCase): def check_basic(self): y=array([1,2,3,4,3,2,1,2,3.0]) x=arange(len(y)) @@ -40,7 +40,7 @@ # make sure interpolated values are on knot points assert_array_almost_equal(y2[::10], y, decimal=5) -class test_order_filt(NumpyTestCase): +class TestOrderFilt(NumpyTestCase): def check_basic(self): assert_array_equal(signal.order_filter([1,2,3],[1,0,1],1), [2,3,2]) Modified: trunk/scipy/signal/tests/test_wavelets.py =================================================================== --- trunk/scipy/signal/tests/test_wavelets.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/signal/tests/test_wavelets.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -5,7 +5,7 @@ from scipy.signal import wavelets restore_path() -class test_wavelets(NumpyTestCase): +class TestWavelets(NumpyTestCase): def check_qmf(self): assert_array_equal(wavelets.qmf([1,1]),[1,-1]) Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -1021,7 +1021,7 @@ [4,0,0], [6,5,0]]) -class test_construct_utils(NumpyTestCase): +class TestConstructUtils(NumpyTestCase): def check_identity(self): a = spidentity(3) b = array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype='d') @@ -1045,7 +1045,7 @@ -class test_coo(NumpyTestCase): +class TestCoo(NumpyTestCase): def check_constructor1(self): row = numpy.array([2, 3, 1, 3, 0, 1, 3, 0, 2, 1, 2]) col = numpy.array([0, 1, 0, 0, 1, 1, 2, 2, 2, 2, 1]) Modified: trunk/scipy/special/tests/test_basic.py =================================================================== --- trunk/scipy/special/tests/test_basic.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/special/tests/test_basic.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -40,7 +40,7 @@ import scipy.special._cephes as cephes restore_path() -class test_cephes(NumpyTestCase): +class TestCephes(NumpyTestCase): def check_airy(self): cephes.airy(0) def check_airye(self): @@ -456,7 +456,7 @@ def check_wofz(self): cephes.wofz(0) -class test_airy(NumpyTestCase): +class TestAiry(NumpyTestCase): def check_airy(self): #This tests the airy function to ensure 8 place accuracy in computation @@ -492,7 +492,7 @@ array([ 0.5357]), array([ 0.7012])),4) -class test_assoc_laguerre(NumpyTestCase): +class TestAssocLaguerre(NumpyTestCase): def check_assoc_laguerre(self): a1 = genlaguerre(11,1) a2 = assoc_laguerre(.2,11,1) @@ -500,11 +500,11 @@ a2 = assoc_laguerre(1,11,1) assert_array_almost_equal(a2,a1(1),8) -class test_besselpoly(NumpyTestCase): +class TestBesselpoly(NumpyTestCase): def check_besselpoly(self): pass -class test_kelvin(NumpyTestCase): +class TestKelvin(NumpyTestCase): def check_bei(self): mbei = bei(2) assert_almost_equal(mbei, 0.9722916273066613,5)#this may not be exact @@ -676,7 +676,7 @@ 16.08312, 20.53068]),4) -class test_bernoulli(NumpyTestCase): +class TestBernoulli(NumpyTestCase): def check_bernoulli(self): brn = bernoulli(5) assert_array_almost_equal(brn,array([1.0000, @@ -686,7 +686,7 @@ -0.0333, 0.0000]),4) -class test_beta(NumpyTestCase): +class TestBeta(NumpyTestCase): def check_beta(self): bet = beta(2,4) betg = (gamma(2)*gamma(4))/gamma(6) @@ -706,7 +706,7 @@ comp = betainc(2,4,y) assert_almost_equal(comp,.5,5) -class test_cheby(NumpyTestCase): +class TestCheby(NumpyTestCase): def check_chebyc(self): C0 = chebyc(0) C1 = chebyc(1) @@ -764,7 +764,7 @@ assert_array_almost_equal(U4.c,[16,0,-12,0,1],13) assert_array_almost_equal(U5.c,[32,0,-32,0,6,0],13) -class test_trigonometric(NumpyTestCase): +class TestTrigonometric(NumpyTestCase): def check_cbrt(self): cb = cbrt(27) cbrl = 27**(1.0/3.0) @@ -837,7 +837,7 @@ snmrl1 = sin(pi/4.0) assert_almost_equal(snm1,snmrl1,8) -class test_tandg(NumpyTestCase): +class TestTandg(NumpyTestCase): def check_tandg(self): tn = tandg(30) @@ -865,7 +865,7 @@ assert_almost_equal(tandg(315), -1.0, 14) assert_almost_equal(tandg(-315), 1.0, 14) -class test_ellip(NumpyTestCase): +class TestEllip(NumpyTestCase): def check_ellipj(self): el = ellipj(0.2,0) rel = [sin(0.2),cos(0.2),1.0,0.20] @@ -901,7 +901,7 @@ assert_almost_equal(eleinc, 0.58823065, 8) -class test_erf(NumpyTestCase): +class TestErf(NumpyTestCase): def check_erf(self): er = erf(.25) @@ -933,7 +933,7 @@ assert_equal(d,b) #makes sure state was returned #assert_equal(d,1-a) -class test_euler(NumpyTestCase): +class TestEuler(NumpyTestCase): def check_euler(self): eu0 = euler(0) eu1 = euler(1) @@ -955,7 +955,7 @@ errmax = max(err) assert_almost_equal(errmax, 0.0, 14) -class test_exp(NumpyTestCase): +class TestExp(NumpyTestCase): def check_exp2(self): ex = exp2(2) exrl = 2**2 @@ -986,7 +986,7 @@ exrl1 = (exp(2)-1,exp(2.1)-1,exp(2.2)-1) assert_array_almost_equal(ex1,exrl1,8) -class test_fresnel(NumpyTestCase): +class TestFresnel(NumpyTestCase): def check_fresnel(self): frs = array(fresnel(.5)) assert_array_almost_equal(frs,array([0.064732432859999287, 0.49234422587144644]),8) @@ -1023,7 +1023,7 @@ assert_array_almost_equal(frs,szo,12) -class test_gamma(NumpyTestCase): +class TestGamma(NumpyTestCase): def check_gamma(self): gam = gamma(5) assert_equal(gam,24.0) @@ -1057,7 +1057,7 @@ rlgam = 1/gamma(8) assert_almost_equal(rgam,rlgam,8) -class test_hankel(NumpyTestCase): +class TestHankel(NumpyTestCase): def check_negv(self): assert_almost_equal(hankel1(-3,2), -hankel1(3,2), 14) @@ -1090,7 +1090,7 @@ hankrl2e = hankel2e(1,.1) assert_almost_equal(hank2e,hankrl2e,8) -class test_hermite(NumpyTestCase): +class TestHermite(NumpyTestCase): def check_hermite(self): H0 = hermite(0) H1 = hermite(1) @@ -1130,7 +1130,7 @@ _gam = cephes.gamma -class test_gegenbauer(NumpyTestCase): +class TestGegenbauer(NumpyTestCase): def check_gegenbauer(self): a = 5*rand()-0.5 @@ -1153,7 +1153,7 @@ 0,15*poch(a,3),0])/15.0,11) -class test_hyper(NumpyTestCase): +class TestHyper(NumpyTestCase): def check_h1vp(self): h1 = h1vp(1,.1) h1real = (jvp(1,.1)+yvp(1,.1)*1j) @@ -1216,7 +1216,7 @@ /(gamma(a)*gamma(2-b))) assert_array_almost_equal(hypu,hprl,12) -class test_bessel(NumpyTestCase): +class TestBessel(NumpyTestCase): def check_i0(self): values = [[0.0, 1.0], [1e-10, 1.0], @@ -1537,7 +1537,7 @@ assert_array_almost_equal(yvp1,yvpr,10) -class test_laguerre(NumpyTestCase): +class TestLaguerre(NumpyTestCase): def check_laguerre(self): lag0 = laguerre(0) lag1 = laguerre(1) @@ -1565,7 +1565,7 @@ # Base polynomials come from Abrahmowitz and Stegan -class test_legendre(NumpyTestCase): +class TestLegendre(NumpyTestCase): def check_legendre(self): leg0 = legendre(0) leg1 = legendre(1) @@ -1581,14 +1581,14 @@ assert_almost_equal(leg5.c,array([63,0,-70,0,15,0])/8.0) -class test_lambda(NumpyTestCase): +class TestLambda(NumpyTestCase): def check_lmbda(self): lam = lmbda(1,.1) lamr = (array([jn(0,.1), 2*jn(1,.1)/.1]), array([jvp(0,.1), -2*jv(1,.1)/.01 + 2*jvp(1,.1)/.1])) assert_array_almost_equal(lam,lamr,8) -class test_log1p(NumpyTestCase): +class TestLog1p(NumpyTestCase): def check_log1p(self): l1p = (log1p(10),log1p(11),log1p(12)) l1prl = (log(11),log(12),log(13)) @@ -1599,7 +1599,7 @@ l1pmrl = (log(2),log(2.1),log(2.2)) assert_array_almost_equal(l1pm,l1pmrl,8) -class test_legendre_functions(NumpyTestCase): +class TestLegendreFunctions(NumpyTestCase): def check_lpmn(self): lp = lpmn(0,2,.5) assert_array_almost_equal(lp,(array([ [ 1.00000 , @@ -1634,7 +1634,7 @@ assert_array_almost_equal(lqf,(array([ 0.5493, -0.7253, -0.8187]), array([ 1.3333, 1.216 , -0.8427])),4) -class test_mathieu(NumpyTestCase): +class TestMathieu(NumpyTestCase): def check_mathieu_a(self): pass @@ -1647,7 +1647,7 @@ pass #same problem as above -class test_fresnel_integral(NumpyTestCase): +class TestFresnelIntegral(NumpyTestCase): def check_modfresnelp(self): pass @@ -1655,7 +1655,7 @@ def check_modfresnelm(self): pass -class test_obl_cv_seq(NumpyTestCase): +class TestOblCvSeq(NumpyTestCase): def check_obl_cv_seq(self): obl = obl_cv_seq(0,3,1) assert_array_almost_equal(obl,array([ -0.348602, @@ -1663,7 +1663,7 @@ 5.486800, 11.492120]),5) -class test_parabolic_cylinder(NumpyTestCase): +class TestParabolicCylinder(NumpyTestCase): def check_pbdn_seq(self): pb = pbdn_seq(1,.1) assert_array_almost_equal(pb,(array([ 0.9975, @@ -1680,7 +1680,7 @@ pbv = pbdv_seq(1,.1) assert_array_almost_equal(pbv,(real(pbn[0]),real(pbn[1])),4) -class test_polygamma(NumpyTestCase): +class TestPolygamma(NumpyTestCase): # from Table 6.2 (pg. 271) of A&S def check_polygamma(self): poly2 = polygamma(2,1) @@ -1688,7 +1688,7 @@ assert_almost_equal(poly2,-2.4041138063,10) assert_almost_equal(poly3,6.4939394023,10) -class test_pro_cv_seq(NumpyTestCase): +class TestProCvSeq(NumpyTestCase): def check_pro_cv_seq(self): prol = pro_cv_seq(0,3,1) assert_array_almost_equal(prol,array([ 0.319000, @@ -1696,12 +1696,12 @@ 6.533471, 12.514462]),5) -class test_psi(NumpyTestCase): +class TestPsi(NumpyTestCase): def check_psi(self): ps = psi(1) assert_almost_equal(ps,-0.57721566490153287,8) -class test_radian(NumpyTestCase): +class TestRadian(NumpyTestCase): def check_radian(self): rad = radian(90,0,0) assert_almost_equal(rad,pi/2.0,5) @@ -1710,7 +1710,7 @@ rad1 = radian(90,1,60) assert_almost_equal(rad1,pi/2+0.0005816135199345904,5) -class test_riccati(NumpyTestCase): +class TestRiccati(NumpyTestCase): def check_riccati_jn(self): jnrl = (sph_jn(1,.2)[0]*.2,sph_jn(1,.2)[0]+sph_jn(1,.2)[1]*.2) ricjn = riccati_jn(1,.2) @@ -1721,7 +1721,7 @@ ricyn = riccati_yn(1,.2) assert_array_almost_equal(ricyn,ynrl,8) -class test_round(NumpyTestCase): +class TestRound(NumpyTestCase): def check_round(self): rnd = map(int,(round(10.1),round(10.4),round(10.5),round(10.6))) @@ -1835,7 +1835,7 @@ assert_array_almost_equal(G4.c,ge4.c,13) assert_array_almost_equal(G5.c,ge5.c,13) -class test_spherical(NumpyTestCase): +class TestSpherical(NumpyTestCase): def check_sph_harm(self): pass Modified: trunk/scipy/special/tests/test_spfun_stats.py =================================================================== --- trunk/scipy/special/tests/test_spfun_stats.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/special/tests/test_spfun_stats.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -5,7 +5,7 @@ from scipy.special import gammaln, multigammaln restore_path() -class test_multigammaln(NumpyTestCase): +class TestMultiGammaLn(NumpyTestCase): def test1(self): a = N.abs(N.random.randn()) assert_array_equal(multigammaln(a, 1), gammaln(a)) Modified: trunk/scipy/stats/models/tests/test_bspline.py =================================================================== --- trunk/scipy/stats/models/tests/test_bspline.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/models/tests/test_bspline.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ import scipy.stats.models.bspline as B -class test_BSpline(NumpyTestCase): +class TestBSpline(NumpyTestCase): def test1(self): b = B.BSpline(N.linspace(0,10,11), x=N.linspace(0,10,101)) Modified: trunk/scipy/stats/models/tests/test_formula.py =================================================================== --- trunk/scipy/stats/models/tests/test_formula.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/models/tests/test_formula.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -11,7 +11,7 @@ from scipy.stats.models import utils, formula, contrast -class test_term(NumpyTestCase): +class TestTerm(NumpyTestCase): def test_init(self): t1 = formula.term("trivial") @@ -47,7 +47,7 @@ f = intercept * t1 self.assertEqual(str(f), str(formula.formula(t1))) -class test_formula(NumpyTestCase): +class TestFormula(NumpyTestCase): def setUp(self): self.X = R.standard_normal((40,10)) Modified: trunk/scipy/stats/models/tests/test_glm.py =================================================================== --- trunk/scipy/stats/models/tests/test_glm.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/models/tests/test_glm.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -11,7 +11,7 @@ W = R.standard_normal -class test_Regression(NumpyTestCase): +class TestRegression(NumpyTestCase): def check_Logistic(self): X = W((40,10)) Modified: trunk/scipy/stats/models/tests/test_regression.py =================================================================== --- trunk/scipy/stats/models/tests/test_regression.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/models/tests/test_regression.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ W = standard_normal -class test_Regression(NumpyTestCase): +class TestRegression(NumpyTestCase): def testOLS(self): X = W((40,10)) Modified: trunk/scipy/stats/models/tests/test_rlm.py =================================================================== --- trunk/scipy/stats/models/tests/test_rlm.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/models/tests/test_rlm.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ W = R.standard_normal -class test_Regression(NumpyTestCase): +class TestRegression(NumpyTestCase): def test_Robust(self): X = W((40,10)) Modified: trunk/scipy/stats/models/tests/test_utils.py =================================================================== --- trunk/scipy/stats/models/tests/test_utils.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/models/tests/test_utils.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from scipy.stats.models import utils -class test_Utils(NumpyTestCase): +class TestUtils(NumpyTestCase): def test_recipr(self): X = N.array([[2,1],[-1,0]]) Modified: trunk/scipy/stats/tests/test_distributions.py =================================================================== --- trunk/scipy/stats/tests/test_distributions.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/tests/test_distributions.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -59,7 +59,7 @@ else: args = str(tuple(1.0+rand(nargs))) exstr = r""" -class test_%s(NumpyTestCase): +class Test%s(NumpyTestCase): def check_cdf(self): D,pval = stats.kstest('%s','',args=%s,N=30) if (pval < %f): @@ -71,7 +71,7 @@ exec exstr -class test_randint(NumpyTestCase): +class TestRandInt(NumpyTestCase): def check_rvs(self): vals = stats.randint.rvs(5,30,size=100) assert(numpy.all(vals < 30) & numpy.all(vals >= 5)) @@ -97,7 +97,7 @@ vals = stats.randint.cdf(x,5,30) assert_array_almost_equal(vals, out, decimal=12) -class test_binom(NumpyTestCase): +class TestBinom(NumpyTestCase): def check_rvs(self): vals = stats.binom.rvs(10, 0.75, size=(2, 50)) assert(numpy.all(vals >= 0) & numpy.all(vals <= 10)) @@ -108,7 +108,7 @@ assert(val.dtype.char in typecodes['AllInteger']) -class test_bernoulli(NumpyTestCase): +class TestBernoulli(NumpyTestCase): def check_rvs(self): vals = stats.bernoulli.rvs(0.75, size=(2, 50)) assert(numpy.all(vals >= 0) & numpy.all(vals <= 1)) @@ -118,7 +118,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_nbinom(NumpyTestCase): +class TestNBinom(NumpyTestCase): def check_rvs(self): vals = stats.nbinom.rvs(10, 0.75, size=(2, 50)) assert(numpy.all(vals >= 0)) @@ -128,7 +128,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_geom(NumpyTestCase): +class TestGeom(NumpyTestCase): def check_rvs(self): vals = stats.geom.rvs(0.75, size=(2, 50)) assert(numpy.all(vals >= 0)) @@ -150,7 +150,7 @@ assert_array_almost_equal(vals_sf,1-expected) -class test_hypergeom(NumpyTestCase): +class TestHypergeom(NumpyTestCase): def check_rvs(self): vals = stats.hypergeom.rvs(20, 10, 3, size=(2, 50)) assert(numpy.all(vals >= 0) & @@ -161,7 +161,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_logser(NumpyTestCase): +class TestLogser(NumpyTestCase): def check_rvs(self): vals = stats.logser.rvs(0.75, size=(2, 50)) assert(numpy.all(vals >= 1)) @@ -171,7 +171,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_poisson(NumpyTestCase): +class TestPoisson(NumpyTestCase): def check_rvs(self): vals = stats.poisson.rvs(0.5, size=(2, 50)) assert(numpy.all(vals >= 0)) @@ -181,7 +181,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_zipf(NumpyTestCase): +class TestZipf(NumpyTestCase): def check_rvs(self): vals = stats.zipf.rvs(1.5, size=(2, 50)) assert(numpy.all(vals >= 1)) @@ -191,7 +191,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_dlaplace(NumpyTestCase): +class TestDLaplace(NumpyTestCase): def check_rvs(self): vals = stats.dlaplace.rvs(1.5 , size=(2, 50)) assert(numpy.shape(vals) == (2, 50)) @@ -200,7 +200,7 @@ assert(isinstance(val, numpy.ndarray)) assert(val.dtype.char in typecodes['AllInteger']) -class test_rv_discrete(NumpyTestCase): +class TestRvDiscrete(NumpyTestCase): def check_rvs(self): states = [-1,0,1,2,3,4] probability = [0.0,0.3,0.4,0.0,0.3,0.0] @@ -211,7 +211,7 @@ for s,p in zip(states,probability): assert abs(sum(x == s)/float(samples) - p) < 0.05 -class test_expon(NumpyTestCase): +class TestExpon(NumpyTestCase): def check_zero(self): assert_equal(stats.expon.pdf(0),1) Modified: trunk/scipy/stats/tests/test_morestats.py =================================================================== --- trunk/scipy/stats/tests/test_morestats.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/tests/test_morestats.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -23,7 +23,7 @@ g9 = [1.002, 0.998, 0.996, 0.995, 0.996, 1.004, 1.004, 0.998, 0.999, 0.991] g10= [0.991, 0.995, 0.984, 0.994, 0.997, 0.997, 0.991, 0.998, 1.004, 0.997] -class test_shapiro(NumpyTestCase): +class TestShapiro(NumpyTestCase): def check_basic(self): x1 = [0.11,7.87,4.61,10.14,7.95,3.14,0.46, 4.43,0.21,4.75,0.71,1.52,3.24, @@ -38,7 +38,7 @@ assert_almost_equal(w,0.9590270,6) assert_almost_equal(pw,0.52460,3) -class test_anderson(NumpyTestCase): +class TestAnderson(NumpyTestCase): def check_normal(self): rs = RandomState(1234567890) x1 = rs.standard_exponential(size=50) @@ -57,7 +57,7 @@ A,crit,sig = scipy.stats.anderson(x2,'expon') assert_array_less(crit[:-1], A) -class test_ansari(NumpyTestCase): +class TestAnsari(NumpyTestCase): def check_small(self): x = [1,2,3,3,4] y = [3,2,6,1,6,1,4,1] @@ -79,7 +79,7 @@ assert_almost_equal(W,10.0,11) assert_almost_equal(pval,0.533333333333333333,7) -class test_bartlett(NumpyTestCase): +class TestBartlett(NumpyTestCase): def check_data(self): args = [] for k in range(1,11): @@ -88,7 +88,7 @@ assert_almost_equal(T,20.78587342806484,7) assert_almost_equal(pval,0.0136358632781,7) -class test_levene(NumpyTestCase): +class TestLevene(NumpyTestCase): def check_data(self): args = [] for k in range(1,11): @@ -97,7 +97,7 @@ assert_almost_equal(W,1.7059176930008939,7) assert_almost_equal(pval,0.0990829755522,7) -class test_binom_test(NumpyTestCase): +class TestBinomTest(NumpyTestCase): def check_data(self): pval = stats.binom_test(100,250) assert_almost_equal(pval,0.0018833009350757682,11) @@ -106,7 +106,7 @@ pval = stats.binom_test([682,243],p=3.0/4) assert_almost_equal(pval,0.38249155957481695,11) -class test_find_repeats(NumpyTestCase): +class TestFindRepeats(NumpyTestCase): def check_basic(self): a = [1,2,3,4,1,2,3,4,1,2,5] res,nums = scipy.stats.find_repeats(a) Modified: trunk/scipy/stats/tests/test_stats.py =================================================================== --- trunk/scipy/stats/tests/test_stats.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/stats/tests/test_stats.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -48,7 +48,7 @@ X8 = X7 * X X9 = X8 * X -class test_round(NumpyTestCase): +class TestRound(NumpyTestCase): """ W.II. ROUND You should get the numbers 1 to 9. Many language compilers, @@ -98,7 +98,7 @@ y=(int(round((3-numpy.exp(numpy.log(numpy.sqrt(2.0)*numpy.sqrt(2.0))))))) assert_equal(y,1) -class test_basicstats(NumpyTestCase): +class TestBasicStats(NumpyTestCase): """ W.II.C. Compute basic statistic on all the variables. The means should be the fifth value of all the variables (case FIVE). @@ -184,7 +184,7 @@ y = scipy.stats.std(ROUND) assert_approx_equal(y, 2.738612788) -class test_nanfunc(NumpyTestCase): +class TestNanFunc(NumpyTestCase): def __init__(self, *args, **kw): NumpyTestCase.__init__(self, *args, **kw) self.X = X.copy() @@ -242,7 +242,7 @@ m = stats.stats.nanmedian(self.Xall) assert numpy.isnan(m) -class test_corr(NumpyTestCase): +class TestCorr(NumpyTestCase): """ W.II.D. Compute a correlation matrix on all the variables. All the correlations, except for ZERO and MISS, shoud be exactly 1. @@ -429,7 +429,7 @@ ### I need to figure out how to do this one. -class test_regression(NumpyTestCase): +class TestRegression(NumpyTestCase): def check_linregressBIGX(self): """ W.II.F. Regress BIG on X. @@ -494,7 +494,7 @@ ################################################## ### Test for sum -class test_gmean(NumpyTestCase): +class TestGMean(NumpyTestCase): def check_1D_list(self): a = (1,2,3,4) @@ -533,7 +533,7 @@ desired = array((v,v,v)) assert_array_almost_equal(desired,actual,decimal=14) -class test_hmean(NumpyTestCase): +class TestHMean(NumpyTestCase): def check_1D_list(self): a = (1,2,3,4) actual= stats.hmean(a) @@ -573,7 +573,7 @@ assert_array_almost_equal(desired1,actual1,decimal=14) -class test_mean(NumpyTestCase): +class TestMean(NumpyTestCase): def check_basic(self): a = [3,4,5,10,-3,-5,6] af = [3.,4,5,10,-3,-5,-6] @@ -610,7 +610,7 @@ A += val assert_almost_equal(stats.mean(a,axis=None),A/(5*3.0*5)) -class test_median(NumpyTestCase): +class TestMedian(NumpyTestCase): def check_basic(self): a1 = [3,4,5,10,-3,-5,6] a2 = [3,-6,-2,8,7,4,2,1] @@ -619,7 +619,7 @@ assert_equal(stats.median(a2),2.5) assert_equal(stats.median(a3),3.5) -class test_percentile(NumpyTestCase): +class TestPercentile(NumpyTestCase): def setUp(self): self.a1 = [3,4,5,10,-3,-5,6] self.a2 = [3,-6,-2,8,7,4,2,1] @@ -646,7 +646,7 @@ [1,1,1]) -class test_std(NumpyTestCase): +class TestStd(NumpyTestCase): def check_basic(self): a = [3,4,5,10,-3,-5,6] b = [3,4,5,10,-3,-5,-6] @@ -665,21 +665,21 @@ assert_array_almost_equal(stats.std(a,axis=1),b2,11) -class test_cmedian(NumpyTestCase): +class TestCMedian(NumpyTestCase): def check_basic(self): data = [1,2,3,1,5,3,6,4,3,2,4,3,5,2.0] assert_almost_equal(stats.cmedian(data,5),3.2916666666666665) assert_almost_equal(stats.cmedian(data,3),3.083333333333333) assert_almost_equal(stats.cmedian(data),3.0020020020020022) -class test_median(NumpyTestCase): +class TestMedian(NumpyTestCase): def check_basic(self): data1 = [1,3,5,2,3,1,19,-10,2,4.0] data2 = [3,5,1,10,23,-10,3,-2,6,8,15] assert_almost_equal(stats.median(data1),2.5) assert_almost_equal(stats.median(data2),5) -class test_mode(NumpyTestCase): +class TestMode(NumpyTestCase): def check_basic(self): data1 = [3,5,1,10,23,3,2,6,8,6,10,6] vals = stats.mode(data1) @@ -687,7 +687,7 @@ assert_almost_equal(vals[1][0],3) -class test_variability(NumpyTestCase): +class TestVariability(NumpyTestCase): """ Comparison numbers are found using R v.1.5.1 note that length(testcase) = 4 """ @@ -767,7 +767,7 @@ -class test_moments(NumpyTestCase): +class TestMoments(NumpyTestCase): """ Comparison numbers are found using R v.1.5.1 note that length(testcase) = 4 @@ -827,7 +827,7 @@ y = scipy.stats.kurtosis(self.testcase,0,0) assert_approx_equal(y,1.64) -class test_threshold(NumpyTestCase): +class TestThreshold(NumpyTestCase): def check_basic(self): a = [-1,2,3,4,5,-1,-2] assert_array_equal(stats.threshold(a),a) Modified: trunk/scipy/weave/tests/test_ast_tools.py =================================================================== --- trunk/scipy/weave/tests/test_ast_tools.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_ast_tools.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from weave_test_utils import * restore_path() -class test_harvest_variables(NumpyTestCase): +class TestHarvestVariables(NumpyTestCase): """ Not much testing going on here, but at least it is a flame test. """ Modified: trunk/scipy/weave/tests/test_blitz_tools.py =================================================================== --- trunk/scipy/weave/tests/test_blitz_tools.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_blitz_tools.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -14,7 +14,7 @@ restore_path() -class test_ast_to_blitz_expr(NumpyTestCase): +class TestAstToBlitzExpr(NumpyTestCase): def generic_test(self,expr,desired): import parser @@ -57,7 +57,7 @@ '-hy(_all,blitz::Range(1,_end),blitz::Range(_beg,Nhy(2)-1-1)));' self.generic_test(expr,desired) -class test_blitz(NumpyTestCase): +class TestBlitz(NumpyTestCase): """* These are long running tests... I'd like to benchmark these things somehow. Modified: trunk/scipy/weave/tests/test_build_tools.py =================================================================== --- trunk/scipy/weave/tests/test_build_tools.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_build_tools.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -12,7 +12,7 @@ def is_writable(val): return os.access(val,os.W_OK) -class test_configure_build_dir(NumpyTestCase): +class TestConfigureBuildDir(NumpyTestCase): def check_default(self): " default behavior is to return current directory " d = build_tools.configure_build_dir() @@ -46,7 +46,7 @@ assert(d == tempfile.gettempdir()) assert(is_writable(d)) -class test_configure_sys_argv(NumpyTestCase): +class TestConfigureSysArgv(NumpyTestCase): def check_simple(self): build_dir = 'build_dir' temp_dir = 'temp_dir' Modified: trunk/scipy/weave/tests/test_c_spec.py =================================================================== --- trunk/scipy/weave/tests/test_c_spec.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_c_spec.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -48,7 +48,7 @@ # Scalar conversion test classes # int, float, complex #---------------------------------------------------------------------------- -class test_int_converter(NumpyTestCase): +class TestIntConverter(NumpyTestCase): compiler = '' def check_type_match_string(self,level=5): s = c_spec.int_converter() @@ -103,7 +103,7 @@ assert( c == 3) -class test_float_converter(NumpyTestCase): +class TestFloatConverter(NumpyTestCase): compiler = '' def check_type_match_string(self,level=5): s = c_spec.float_converter() @@ -158,7 +158,7 @@ c = test(b) assert( c == 3.) -class test_complex_converter(NumpyTestCase): +class TestComplexConverter(NumpyTestCase): compiler = '' def check_type_match_string(self,level=5): s = c_spec.complex_converter() @@ -216,7 +216,7 @@ # File conversion tests #---------------------------------------------------------------------------- -class test_file_converter(NumpyTestCase): +class TestFileConverter(NumpyTestCase): compiler = '' def check_py_to_file(self,level=5): import tempfile @@ -250,14 +250,14 @@ # Instance conversion tests #---------------------------------------------------------------------------- -class test_instance_converter(NumpyTestCase): +class TestInstanceConverter(NumpyTestCase): pass #---------------------------------------------------------------------------- # Callable object conversion tests #---------------------------------------------------------------------------- -class test_callable_converter(NumpyTestCase): +class TestCallableConverter(NumpyTestCase): compiler='' def check_call_function(self,level=5): import string @@ -277,7 +277,7 @@ desired = func(search_str,sub_str) assert(desired == actual) -class test_sequence_converter(NumpyTestCase): +class TestSequenceConverter(NumpyTestCase): compiler = '' def check_convert_to_dict(self,level=5): d = {} @@ -292,7 +292,7 @@ t = () inline_tools.inline("",['t'],compiler=self.compiler,force=1) -class test_string_converter(NumpyTestCase): +class TestStringConverter(NumpyTestCase): compiler = '' def check_type_match_string(self,level=5): s = c_spec.string_converter() @@ -347,7 +347,7 @@ c = test(b) assert( c == 'hello') -class test_list_converter(NumpyTestCase): +class TestListConverter(NumpyTestCase): compiler = '' def check_type_match_bad(self,level=5): s = c_spec.list_converter() @@ -458,7 +458,7 @@ print 'python:', t2 - t1 assert( sum1 == sum2 and sum1 == sum3) -class test_tuple_converter(NumpyTestCase): +class TestTupleConverter(NumpyTestCase): compiler = '' def check_type_match_bad(self,level=5): s = c_spec.tuple_converter() @@ -511,7 +511,7 @@ assert( c == ('hello',None)) -class test_dict_converter(NumpyTestCase): +class TestDictConverter(NumpyTestCase): """ Base Class for dictionary conversion tests. """ Modified: trunk/scipy/weave/tests/test_catalog.py =================================================================== --- trunk/scipy/weave/tests/test_catalog.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_catalog.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -11,7 +11,7 @@ restore_path() -class test_default_dir(NumpyTestCase): +class TestDefaultDir(NumpyTestCase): def check_is_writable(self): path = catalog.default_dir() name = os.path.join(path,'dummy_catalog') @@ -22,10 +22,10 @@ test_file.close() os.remove(name) -class test_os_dependent_catalog_name(NumpyTestCase): +class TestOsDependentCatalogName(NumpyTestCase): pass -class test_catalog_path(NumpyTestCase): +class TestCatalogPath(NumpyTestCase): def check_default(self): in_path = catalog.default_dir() path = catalog.catalog_path(in_path) @@ -64,7 +64,7 @@ path = catalog.catalog_path(in_path) assert (path is None) -class test_get_catalog(NumpyTestCase): +class TestGetCatalog(NumpyTestCase): """ This only tests whether new catalogs are created correctly. And whether non-existent return None correctly with read mode. Putting catalogs in the right place is all tested with @@ -98,7 +98,7 @@ self.remove_dir(pardir) assert(cat is not None) -class test_catalog(NumpyTestCase): +class TestCatalog(NumpyTestCase): def clear_environ(self): if os.environ.has_key('PYTHONCOMPILED'): Modified: trunk/scipy/weave/tests/test_ext_tools.py =================================================================== --- trunk/scipy/weave/tests/test_ext_tools.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_ext_tools.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -17,7 +17,7 @@ build_dir = empty_temp_dir() print 'building extensions here:', build_dir -class test_ext_module(NumpyTestCase): +class TestExtModule(NumpyTestCase): #should really do some testing of where modules end up def check_simple(self,level=5): """ Simplest possible module """ @@ -94,7 +94,7 @@ c,d = ext_return_tuple.test(a) assert(c==a and d == a+1) -class test_ext_function(NumpyTestCase): +class TestExtFunction(NumpyTestCase): #should really do some testing of where modules end up def check_simple(self,level=5): """ Simplest possible function """ @@ -107,7 +107,7 @@ import simple_ext_function simple_ext_function.test() -class test_assign_variable_types(NumpyTestCase): +class TestAssignVariableTypes(NumpyTestCase): def check_assign_variable_types(self): try: from numpy.numerix import arange, Float32, Float64 Modified: trunk/scipy/weave/tests/test_inline_tools.py =================================================================== --- trunk/scipy/weave/tests/test_inline_tools.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_inline_tools.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ from test_scxx import * restore_path() -class test_inline(NumpyTestCase): +class TestInline(NumpyTestCase): """ These are long running tests... I'd like to benchmark these things somehow. Modified: trunk/scipy/weave/tests/test_numpy_scalar_spec.py =================================================================== --- trunk/scipy/weave/tests/test_numpy_scalar_spec.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_numpy_scalar_spec.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -52,7 +52,7 @@ # int, float, complex #---------------------------------------------------------------------------- -class test_numpy_complex_scalar_converter(NumpyTestCase): +class TestNumpyComplexScalarConverter(NumpyTestCase): compiler = '' def setUp(self): Modified: trunk/scipy/weave/tests/test_scxx_dict.py =================================================================== --- trunk/scipy/weave/tests/test_scxx_dict.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_scxx_dict.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -9,7 +9,7 @@ restore_path() -class test_dict_construct(NumpyTestCase): +class TestDictConstruct(NumpyTestCase): #------------------------------------------------------------------------ # Check that construction from basic types is allowed and have correct # reference counts @@ -25,7 +25,7 @@ assert res == {} -class test_dict_has_key(NumpyTestCase): +class TestDictHasKey(NumpyTestCase): def check_obj(self,level=5): class foo: pass @@ -89,7 +89,7 @@ res = inline_tools.inline(code,['a']) assert not res -class test_dict_get_item_op(NumpyTestCase): +class TestDictGetItemOp(NumpyTestCase): def generic_get(self,code,args=['a']): a = {} @@ -132,7 +132,7 @@ except KeyError: pass -class test_dict_set_operator(NumpyTestCase): +class TestDictSetOperator(NumpyTestCase): def generic_new(self,key,val): # test that value is set correctly and that reference counts # on dict, key, and val are being handled correctly. @@ -199,7 +199,7 @@ key,val = foo(),12345 self.generic_overwrite(key,val) -class test_dict_del(NumpyTestCase): +class TestDictDel(NumpyTestCase): def generic(self,key): # test that value is set correctly and that reference counts # on dict, key, are being handled correctly. after deletion, @@ -233,7 +233,7 @@ key = foo() self.generic(key) -class test_dict_others(NumpyTestCase): +class TestDictOthers(NumpyTestCase): def check_clear(self,level=5): a = {} a["hello"] = 1 Modified: trunk/scipy/weave/tests/test_scxx_object.py =================================================================== --- trunk/scipy/weave/tests/test_scxx_object.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_scxx_object.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -8,7 +8,7 @@ from weave import inline_tools restore_path() -class test_object_construct(NumpyTestCase): +class TestObjectConstruct(NumpyTestCase): #------------------------------------------------------------------------ # Check that construction from basic types is allowed and have correct # reference counts @@ -66,7 +66,7 @@ assert_equal(sys.getrefcount(res),2) assert_equal(res,"hello") -class test_object_print(NumpyTestCase): +class TestObjectPrint(NumpyTestCase): #------------------------------------------------------------------------ # Check the object print protocol. #------------------------------------------------------------------------ @@ -101,7 +101,7 @@ ## pass -class test_object_cast(NumpyTestCase): +class TestObjectCast(NumpyTestCase): def check_int_cast(self,level=5): code = """ py::object val = 1; @@ -147,7 +147,7 @@ def __str__(self): return "b" -class test_object_hasattr(NumpyTestCase): +class TestObjectHasattr(NumpyTestCase): def check_string(self,level=5): a = foo() a.b = 12345 @@ -203,7 +203,7 @@ res = inline_tools.inline(code,['a']) assert res -class test_object_attr(NumpyTestCase): +class TestObjectAttr(NumpyTestCase): def generic_attr(self,code,args=['a']): a = foo() @@ -261,7 +261,7 @@ assert_equal(res,"bar results") assert_equal(first,second) -class test_object_set_attr(NumpyTestCase): +class TestObjectSetAttr(NumpyTestCase): def generic_existing(self, code, desired): args = ['a'] @@ -325,7 +325,7 @@ """ self.generic_existing(code,"hello") -class test_object_del(NumpyTestCase): +class TestObjectDel(NumpyTestCase): def generic(self, code): args = ['a'] a = foo() @@ -348,7 +348,7 @@ """ self.generic(code) -class test_object_cmp(NumpyTestCase): +class TestObjectCmp(NumpyTestCase): def check_equal(self,level=5): a,b = 1,1 res = inline_tools.inline('return_val = (a == b);',['a','b']) @@ -411,7 +411,7 @@ res = inline_tools.inline(code,['a']) assert_equal(res,(a == "hello")) -class test_object_repr(NumpyTestCase): +class TestObjectRepr(NumpyTestCase): def check_repr(self,level=5): class foo: def __str__(self): @@ -427,7 +427,7 @@ assert_equal(first,second) assert_equal(res,"repr return") -class test_object_str(NumpyTestCase): +class TestObjectStr(NumpyTestCase): def check_str(self,level=5): class foo: def __str__(self): @@ -444,7 +444,7 @@ print res assert_equal(res,"str return") -class test_object_unicode(NumpyTestCase): +class TestObjectUnicode(NumpyTestCase): # This ain't going to win awards for test of the year... def check_unicode(self,level=5): class foo: @@ -461,7 +461,7 @@ assert_equal(first,second) assert_equal(res,"unicode") -class test_object_is_callable(NumpyTestCase): +class TestObjectIsCallable(NumpyTestCase): def check_true(self,level=5): class foo: def __call__(self): @@ -476,7 +476,7 @@ res = inline_tools.inline('return_val = a.is_callable();',['a']) assert not res -class test_object_call(NumpyTestCase): +class TestObjectCall(NumpyTestCase): def check_noargs(self,level=5): def foo(): return (1,2,3) @@ -532,7 +532,7 @@ # first should == second, but the weird refcount error assert_equal(second,third) -class test_object_mcall(NumpyTestCase): +class TestObjectMcall(NumpyTestCase): def check_noargs(self,level=5): a = foo() res = inline_tools.inline('return_val = a.mcall("bar");',['a']) @@ -626,7 +626,7 @@ # first should == second, but the weird refcount error assert_equal(second,third) -class test_object_hash(NumpyTestCase): +class TestObjectHash(NumpyTestCase): def check_hash(self,level=5): class foo: def __hash__(self): @@ -636,7 +636,7 @@ print 'hash:', res assert_equal(res,123) -class test_object_is_true(NumpyTestCase): +class TestObjectIsTrue(NumpyTestCase): def check_true(self,level=5): class foo: pass @@ -648,7 +648,7 @@ res = inline_tools.inline('return_val = a.is_true();',['a']) assert_equal(res,0) -class test_object_is_true(NumpyTestCase): +class TestObjectIsTrue(NumpyTestCase): def check_false(self,level=5): class foo: pass @@ -660,7 +660,7 @@ res = inline_tools.inline('return_val = a.mcall("not");',['a']) assert_equal(res,1) -class test_object_type(NumpyTestCase): +class TestObjectType(NumpyTestCase): def check_type(self,level=5): class foo: pass @@ -668,7 +668,7 @@ res = inline_tools.inline('return_val = a.type();',['a']) assert_equal(res,type(a)) -class test_object_size(NumpyTestCase): +class TestObjectSize(NumpyTestCase): def check_size(self,level=5): class foo: def __len__(self): @@ -692,7 +692,7 @@ assert_equal(res,len(a)) from UserList import UserList -class test_object_set_item_op_index(NumpyTestCase): +class TestObjectSetItemOpIndex(NumpyTestCase): def check_list_refcount(self,level=5): a = UserList([1,2,3]) # temporary refcount fix until I understand why it incs by one. @@ -727,7 +727,7 @@ assert_equal(a[1],1+1j) from UserDict import UserDict -class test_object_set_item_op_key(NumpyTestCase): +class TestObjectSetItemOpKey(NumpyTestCase): def check_key_refcount(self,level=5): a = UserDict() code = """ Modified: trunk/scipy/weave/tests/test_size_check.py =================================================================== --- trunk/scipy/weave/tests/test_size_check.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_size_check.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -11,7 +11,7 @@ empty = array(()) -class test_make_same_length(NumpyTestCase): +class TestMakeSameLength(NumpyTestCase): def generic_test(self,x,y,desired): actual = size_check.make_same_length(x,y) @@ -39,7 +39,7 @@ desired = array((1,2,3)),array((1,1,2)) self.generic_test(x,y,desired) -class test_binary_op_size(NumpyTestCase): +class TestBinaryOpSize(NumpyTestCase): def generic_test(self,x,y,desired): actual = size_check.binary_op_size(x,y) desired = desired @@ -115,7 +115,7 @@ def desired_type(self,val): return size_check.dummy_array(array(val),1) -class test_dummy_array_indexing(NumpyTestCase): +class TestDummyArrayIndexing(NumpyTestCase): def generic_test(self,ary,expr,desired): a = size_check.dummy_array(ary) actual = eval(expr).shape @@ -267,7 +267,7 @@ except IndexError: pass -class test_reduction(NumpyTestCase): +class TestReduction(NumpyTestCase): def check_1d_0(self): a = ones((5,)) actual = size_check.reduction(a,0) @@ -301,7 +301,7 @@ except ValueError: pass -class test_expressions(NumpyTestCase): +class TestExpressions(NumpyTestCase): def generic_test(self,expr,desired,**kw): import parser ast_list = parser.expr(expr).tolist() Modified: trunk/scipy/weave/tests/test_slice_handler.py =================================================================== --- trunk/scipy/weave/tests/test_slice_handler.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_slice_handler.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -22,7 +22,7 @@ pprint.pprint(desired,msg) raise AssertionError, msg.getvalue() -class test_build_slice_atom(NumpyTestCase): +class TestBuildSliceAtom(NumpyTestCase): def generic_test(self,slice_vars,desired): pos = slice_vars['pos'] ast_list = slice_handler.build_slice_atom(slice_vars,pos) @@ -34,7 +34,7 @@ desired = 'slice(1,2-1)' self.generic_test(slice_vars,desired) -class test_slice(NumpyTestCase): +class TestSlice(NumpyTestCase): def generic_test(self,suite_string,desired): import parser @@ -135,7 +135,7 @@ out = string.replace(out,"\n","") return out -class test_transform_slices(NumpyTestCase): +class TestTransformSlices(NumpyTestCase): def generic_test(self,suite_string,desired): import parser ast_list = parser.suite(suite_string).tolist() Modified: trunk/scipy/weave/tests/test_standard_array_spec.py =================================================================== --- trunk/scipy/weave/tests/test_standard_array_spec.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_standard_array_spec.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -28,7 +28,7 @@ pprint.pprint(desired,msg) raise AssertionError, msg.getvalue() -class test_array_converter(NumpyTestCase): +class TestArrayConverter(NumpyTestCase): def check_type_match_string(self): s = standard_array_spec.array_converter() assert( not s.type_match('string') ) Modified: trunk/scipy/weave/tests/test_wx_spec.py =================================================================== --- trunk/scipy/weave/tests/test_wx_spec.py 2007-10-02 07:40:02 UTC (rev 3387) +++ trunk/scipy/weave/tests/test_wx_spec.py 2007-10-02 09:00:27 UTC (rev 3388) @@ -15,7 +15,7 @@ import wx -class test_wx_converter(NumpyTestCase): +class TestWxConverter(NumpyTestCase): def setUp(self): self.app = wx.App() self.s = wx_spec.wx_converter() From scipy-svn at scipy.org Tue Oct 2 20:06:43 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 2 Oct 2007 19:06:43 -0500 (CDT) Subject: [Scipy-svn] r3389 - trunk/scipy/io Message-ID: <20071003000643.1323539C16B@new.scipy.org> Author: chris.burns Date: 2007-10-02 19:06:40 -0500 (Tue, 02 Oct 2007) New Revision: 3389 Modified: trunk/scipy/io/datasource.py Log: Documentation updates and code refactoring. Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-02 09:00:27 UTC (rev 3388) +++ trunk/scipy/io/datasource.py 2007-10-03 00:06:40 UTC (rev 3389) @@ -19,7 +19,7 @@ file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file} def iszip(filename): - """Is filename a zip file. + """Test if the given file is a zip file. *Parameters*: @@ -37,17 +37,17 @@ return ext in zipexts def unzip(filename): - """Unzip filename into another file. + """Unzip the given file and return the path object to the new file. *Parameters*: - filename : {string} + filename : string Filename to unzip. *Returns*: - string - Name of the unzipped file. + path + Path object of the unzipped file. """ @@ -75,9 +75,11 @@ """ - return mode.find("w")>-1 or mode.find("+")>-1 + _writemodes = ("w", "+", "a") + for c in mode: + if c in _writemodes: return True + return False - def splitzipext(filename): """Return a tuple containing the filename and the zip extension separated. @@ -105,36 +107,39 @@ def isurl(pathstr): - """ - Check whether a given string can be parsed as a URL. + """Test whether a given string can be parsed as a URL. - :Parameters: - `pathstr` : string + *Parameters* + pathstr : {string} The string to be checked. - :Returns: ``bool`` + *Returns*: + bool + Results of test. + """ - scheme, netloc, _, _, _, _ = urlparse(pathstr) + + scheme, netloc, _tmp, _tmp, _tmp, _tmp = urlparse(pathstr) return bool(scheme and netloc) +def ensuredirs(directory): + """Ensure that the given directory path actually exists. + If the directory does not exist, it is created. + *Parameters*: + directory : {path object} + + *Returns*: + None -def ensuredirs(directory): """ - Ensure that the given directory path actually exists. - If it doesn't, create it. - :Returns: ``None`` - """ if not isinstance(directory, path): directory = path(directory) if not directory.exists(): directory.makedirs() - - - class Cache (object): """A file cache. From scipy-svn at scipy.org Tue Oct 2 21:46:32 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 2 Oct 2007 20:46:32 -0500 (CDT) Subject: [Scipy-svn] r3390 - trunk/scipy/sandbox/maskedarray Message-ID: <20071003014632.294BC39C04D@new.scipy.org> Author: pierregm Date: 2007-10-02 20:46:24 -0500 (Tue, 02 Oct 2007) New Revision: 3390 Modified: trunk/scipy/sandbox/maskedarray/core.py trunk/scipy/sandbox/maskedarray/extras.py trunk/scipy/sandbox/maskedarray/morestats.py trunk/scipy/sandbox/maskedarray/mrecords.py trunk/scipy/sandbox/maskedarray/mstats.py trunk/scipy/sandbox/maskedarray/testutils.py Log: Updated documentation. core : the transpose/reshape/resize functions now return a MaskedArray instance systematically. Modified: trunk/scipy/sandbox/maskedarray/core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/core.py 2007-10-03 00:06:40 UTC (rev 3389) +++ trunk/scipy/sandbox/maskedarray/core.py 2007-10-03 01:46:24 UTC (rev 3390) @@ -21,6 +21,8 @@ __revision__ = "$Revision$" __date__ = '$Date$' +__docformat__ = "restructuredtext en" + __all__ = ['MAError', 'MaskType', 'MaskedArray', 'bool_', 'complex_', 'float_', 'int_', 'object_', 'abs', 'absolute', 'add', 'all', 'allclose', 'allequal', 'alltrue', @@ -119,7 +121,8 @@ min_filler.update([(numpy.float128, numpy.inf)]) def default_fill_value(obj): - "Calculates the default fill value for an object `obj`." + """Calculates the default fill value for the argument object. + """ if hasattr(obj,'dtype'): defval = default_filler[obj.dtype.kind] elif isinstance(obj, numeric.dtype): @@ -137,7 +140,7 @@ return defval def minimum_fill_value(obj): - "Calculates the default fill value suitable for taking the minimum of `obj`." + "Calculates the default fill value suitable for taking the minimum of ``obj``." if hasattr(obj, 'dtype'): objtype = obj.dtype filler = min_filler[objtype] @@ -156,7 +159,7 @@ raise TypeError, 'Unsuitable type for calculating minimum.' def maximum_fill_value(obj): - "Calculates the default fill value suitable for taking the maximum of `obj`." + "Calculates the default fill value suitable for taking the maximum of ``obj``." if hasattr(obj, 'dtype'): objtype = obj.dtype filler = max_filler[objtype] @@ -175,13 +178,19 @@ raise TypeError, 'Unsuitable type for calculating minimum.' def set_fill_value(a, fill_value): - "Sets the fill value of `a` if it is a masked array." + """Sets the filling value of a, if a is a masked array. +Otherwise, does nothing. + +*Returns*: + None + """ if isinstance(a, MaskedArray): a.set_fill_value(fill_value) + return def get_fill_value(a): - """Returns the fill value of `a`, if any. - Otherwise, returns the default fill value for that type. + """Returns the filling value of a, if any. +Otherwise, returns the default filling value for that type. """ if isinstance(a, MaskedArray): result = a.fill_value @@ -190,7 +199,8 @@ return result def common_fill_value(a, b): - "Returns the common fill_value of `a` and `b`, if any, or `None`." + """Returns the common filling value of a and b, if any. + If a and b have different filling values, returns None.""" t1 = get_fill_value(a) t2 = get_fill_value(b) if t1 == t2: @@ -201,8 +211,17 @@ def filled(a, value = None): """Returns a as an array with masked data replaced by value. If value is None, get_fill_value(a) is used instead. +If a is already a ndarray, a itself is returned. -If a is already a ndarray, a itself is returned. +*Parameters*: + a : {var} + An input object. + value : {var}, optional + Filling value. If not given, the output of get_fill_value(a) is used instead. + +*Returns*: + A ndarray. + """ if hasattr(a, 'filled'): return a.filled(value) @@ -215,8 +234,8 @@ #####-------------------------------------------------------------------------- def get_masked_subclass(*arrays): - """Returns the youngest subclass of MaskedArray from a list of arrays, - or MaskedArray. In case of siblings, the first takes over.""" + """Returns the youngest subclass of MaskedArray from a list of (masked) arrays. +In case of siblings, the first takes over.""" if len(arrays) == 1: arr = arrays[0] if isinstance(arr, MaskedArray): @@ -237,11 +256,11 @@ def get_data(a, subok=True): """Returns the _data part of a (if any), or a as a ndarray. -:Parameters: - a : ndarray +*Parameters* : + a : {ndarray} A ndarray or a subclass of. - subok : boolean *[True]* - Whether to force a to a 'pure' ndarray (False) or to return a subclass + subok : {boolean} + Whether to force the output to a 'pure' ndarray (False) or to return a subclass of ndarray if approriate (True). """ data = getattr(a, '_data', numpy.array(a, subok=subok)) @@ -252,17 +271,19 @@ def fix_invalid(a, copy=False, fill_value=None): """Returns (a copy of) a where invalid data (nan/inf) are masked and replaced - by fill_value. - If fill_value is None, a.fill_value is used instead. +by fill_value. -:Parameters: - a : ndarray - A ndarray or a subclass of. - copy : boolean *[False]* +*Parameters*: + a : {ndarray} + A (subclass of) ndarray. + copy : {boolean} Whether to use a copy of a (True) or to fix a in place (False). - fill_value : var *[None]* - Value used for fixing invalid data. If None, use a default based on the - datatype. + fill_value : {var}, optional + Value used for fixing invalid data. + If not given, the output of get_fill_value(a) is used instead. + +*Returns* : + MaskedArray object """ a = masked_array(a, copy=copy, subok=True) invalid = (numpy.isnan(a._data) | numpy.isinf(a._data)) @@ -280,9 +301,9 @@ ufunc_domain = {} ufunc_fills = {} -class domain_check_interval: - """Defines a valid interval, -so that `domain_check_interval(a,b)(x) = true` where `x < a` or `x > b`.""" +class _DomainCheckInterval: + """Defines a valid interval, so that : +``domain_check_interval(a,b)(x) = true`` where ``x < a`` or ``x > b``.""" def __init__(self, a, b): "domain_check_interval(a,b)(x) = true where x < a or y > b" if (a > b): @@ -295,9 +316,9 @@ return umath.logical_or(umath.greater (x, self.b), umath.less(x, self.a)) #............................ -class domain_tan: - """Defines a valid interval for the `tan` function, -so that `domain_tan(eps) = True where `abs(cos(x)) < eps`""" +class _DomainTan: + """Defines a valid interval for the `tan` function, so that: +``domain_tan(eps) = True`` where ``abs(cos(x)) < eps``""" def __init__(self, eps): "domain_tan(eps) = true where abs(cos(x)) < eps)" self.eps = eps @@ -305,27 +326,27 @@ "Executes the call behavior." return umath.less(umath.absolute(umath.cos(x)), self.eps) #............................ -class domain_safe_divide: +class _DomainSafeDivide: """Defines a domain for safe division.""" def __init__ (self, tolerance=divide_tolerance): self.tolerance = tolerance def __call__ (self, a, b): return umath.absolute(a) * self.tolerance >= umath.absolute(b) #............................ -class domain_greater: - "domain_greater(v)(x) = true where x <= v" +class _DomainGreater: + "DomainGreater(v)(x) = true where x <= v" def __init__(self, critical_value): - "domain_greater(v)(x) = true where x <= v" + "DomainGreater(v)(x) = true where x <= v" self.critical_value = critical_value def __call__ (self, x): "Executes the call behavior." return umath.less_equal(x, self.critical_value) #............................ -class domain_greater_equal: - "domain_greater_equal(v)(x) = true where x < v" +class _DomainGreaterEqual: + "DomainGreaterEqual(v)(x) = true where x < v" def __init__(self, critical_value): - "domain_greater_equal(v)(x) = true where x < v" + "DomainGreaterEqual(v)(x) = true where x < v" self.critical_value = critical_value def __call__ (self, x): @@ -333,9 +354,8 @@ return umath.less(x, self.critical_value) #.............................................................................. -class masked_unary_operation: - """Defines masked version of unary operations, -where invalid values are pre-masked. +class _MaskedUnaryOperation: + """Defines masked version of unary operations, where invalid values are pre-masked. :IVariables: f : function. @@ -343,7 +363,7 @@ domain : Default domain *[None]*. """ def __init__ (self, mufunc, fill=0, domain=None): - """ masked_unary_operation(aufunc, fill=0, domain=None) + """ _MaskedUnaryOperation(aufunc, fill=0, domain=None) aufunc(fill) must be defined self(x) returns aufunc(x) with masked values where domain(x) is true or getmask(x) is true. @@ -384,7 +404,7 @@ return "Masked version of %s. [Invalid values are masked]" % str(self.f) #.............................................................................. -class masked_binary_operation: +class _MaskedBinaryOperation: """Defines masked version of binary operations, where invalid values are pre-masked. @@ -478,10 +498,9 @@ return "Masked version of " + str(self.f) #.............................................................................. -class domained_binary_operation: +class _DomainedBinaryOperation: """Defines binary operations that have a domain, like divide. -These are complicated so they are a separate class. They have no reduce, outer or accumulate. :IVariables: @@ -528,70 +547,75 @@ #.............................................................................. # Unary ufuncs -exp = masked_unary_operation(umath.exp) -conjugate = masked_unary_operation(umath.conjugate) -sin = masked_unary_operation(umath.sin) -cos = masked_unary_operation(umath.cos) -tan = masked_unary_operation(umath.tan) -arctan = masked_unary_operation(umath.arctan) -arcsinh = masked_unary_operation(umath.arcsinh) -sinh = masked_unary_operation(umath.sinh) -cosh = masked_unary_operation(umath.cosh) -tanh = masked_unary_operation(umath.tanh) -abs = absolute = masked_unary_operation(umath.absolute) -fabs = masked_unary_operation(umath.fabs) -negative = masked_unary_operation(umath.negative) -floor = masked_unary_operation(umath.floor) -ceil = masked_unary_operation(umath.ceil) -around = masked_unary_operation(fromnumeric.round_) -logical_not = masked_unary_operation(umath.logical_not) +exp = _MaskedUnaryOperation(umath.exp) +conjugate = _MaskedUnaryOperation(umath.conjugate) +sin = _MaskedUnaryOperation(umath.sin) +cos = _MaskedUnaryOperation(umath.cos) +tan = _MaskedUnaryOperation(umath.tan) +arctan = _MaskedUnaryOperation(umath.arctan) +arcsinh = _MaskedUnaryOperation(umath.arcsinh) +sinh = _MaskedUnaryOperation(umath.sinh) +cosh = _MaskedUnaryOperation(umath.cosh) +tanh = _MaskedUnaryOperation(umath.tanh) +abs = absolute = _MaskedUnaryOperation(umath.absolute) +fabs = _MaskedUnaryOperation(umath.fabs) +negative = _MaskedUnaryOperation(umath.negative) +floor = _MaskedUnaryOperation(umath.floor) +ceil = _MaskedUnaryOperation(umath.ceil) +around = _MaskedUnaryOperation(fromnumeric.round_) +logical_not = _MaskedUnaryOperation(umath.logical_not) # Domained unary ufuncs ....................................................... -sqrt = masked_unary_operation(umath.sqrt, 0.0, domain_greater_equal(0.0)) -log = masked_unary_operation(umath.log, 1.0, domain_greater(0.0)) -log10 = masked_unary_operation(umath.log10, 1.0, domain_greater(0.0)) -tan = masked_unary_operation(umath.tan, 0.0, domain_tan(1.e-35)) -arcsin = masked_unary_operation(umath.arcsin, 0.0, - domain_check_interval(-1.0, 1.0)) -arccos = masked_unary_operation(umath.arccos, 0.0, - domain_check_interval(-1.0, 1.0)) -arccosh = masked_unary_operation(umath.arccosh, 1.0, domain_greater_equal(1.0)) -arctanh = masked_unary_operation(umath.arctanh, 0.0, - domain_check_interval(-1.0+1e-15, 1.0-1e-15)) +sqrt = _MaskedUnaryOperation(umath.sqrt, 0.0, + _DomainGreaterEqual(0.0)) +log = _MaskedUnaryOperation(umath.log, 1.0, + _DomainGreater(0.0)) +log10 = _MaskedUnaryOperation(umath.log10, 1.0, + _DomainGreater(0.0)) +tan = _MaskedUnaryOperation(umath.tan, 0.0, + _DomainTan(1.e-35)) +arcsin = _MaskedUnaryOperation(umath.arcsin, 0.0, + _DomainCheckInterval(-1.0, 1.0)) +arccos = _MaskedUnaryOperation(umath.arccos, 0.0, + _DomainCheckInterval(-1.0, 1.0)) +arccosh = _MaskedUnaryOperation(umath.arccosh, 1.0, + _DomainGreaterEqual(1.0)) +arctanh = _MaskedUnaryOperation(umath.arctanh, 0.0, + _DomainCheckInterval(-1.0+1e-15, 1.0-1e-15)) # Binary ufuncs ............................................................... -add = masked_binary_operation(umath.add) -subtract = masked_binary_operation(umath.subtract) -multiply = masked_binary_operation(umath.multiply, 1, 1) -arctan2 = masked_binary_operation(umath.arctan2, 0.0, 1.0) -equal = masked_binary_operation(umath.equal) +add = _MaskedBinaryOperation(umath.add) +subtract = _MaskedBinaryOperation(umath.subtract) +multiply = _MaskedBinaryOperation(umath.multiply, 1, 1) +arctan2 = _MaskedBinaryOperation(umath.arctan2, 0.0, 1.0) +equal = _MaskedBinaryOperation(umath.equal) equal.reduce = None -not_equal = masked_binary_operation(umath.not_equal) +not_equal = _MaskedBinaryOperation(umath.not_equal) not_equal.reduce = None -less_equal = masked_binary_operation(umath.less_equal) +less_equal = _MaskedBinaryOperation(umath.less_equal) less_equal.reduce = None -greater_equal = masked_binary_operation(umath.greater_equal) +greater_equal = _MaskedBinaryOperation(umath.greater_equal) greater_equal.reduce = None -less = masked_binary_operation(umath.less) +less = _MaskedBinaryOperation(umath.less) less.reduce = None -greater = masked_binary_operation(umath.greater) +greater = _MaskedBinaryOperation(umath.greater) greater.reduce = None -logical_and = masked_binary_operation(umath.logical_and) -alltrue = masked_binary_operation(umath.logical_and, 1, 1).reduce -logical_or = masked_binary_operation(umath.logical_or) +logical_and = _MaskedBinaryOperation(umath.logical_and) +alltrue = _MaskedBinaryOperation(umath.logical_and, 1, 1).reduce +logical_or = _MaskedBinaryOperation(umath.logical_or) sometrue = logical_or.reduce -logical_xor = masked_binary_operation(umath.logical_xor) -bitwise_and = masked_binary_operation(umath.bitwise_and) -bitwise_or = masked_binary_operation(umath.bitwise_or) -bitwise_xor = masked_binary_operation(umath.bitwise_xor) -hypot = masked_binary_operation(umath.hypot) +logical_xor = _MaskedBinaryOperation(umath.logical_xor) +bitwise_and = _MaskedBinaryOperation(umath.bitwise_and) +bitwise_or = _MaskedBinaryOperation(umath.bitwise_or) +bitwise_xor = _MaskedBinaryOperation(umath.bitwise_xor) +hypot = _MaskedBinaryOperation(umath.hypot) # Domained binary ufuncs ...................................................... -divide = domained_binary_operation(umath.divide, domain_safe_divide(), 0, 1) -true_divide = domained_binary_operation(umath.true_divide, - domain_safe_divide(), 0, 1) -floor_divide = domained_binary_operation(umath.floor_divide, - domain_safe_divide(), 0, 1) -remainder = domained_binary_operation(umath.remainder, - domain_safe_divide(), 0, 1) -fmod = domained_binary_operation(umath.fmod, domain_safe_divide(), 0, 1) +divide = _DomainedBinaryOperation(umath.divide, _DomainSafeDivide(), 0, 1) +true_divide = _DomainedBinaryOperation(umath.true_divide, + _DomainSafeDivide(), 0, 1) +floor_divide = _DomainedBinaryOperation(umath.floor_divide, + _DomainSafeDivide(), 0, 1) +remainder = _DomainedBinaryOperation(umath.remainder, + _DomainSafeDivide(), 0, 1) +fmod = _DomainedBinaryOperation(umath.fmod, _DomainSafeDivide(), 0, 1) #####-------------------------------------------------------------------------- @@ -599,12 +623,13 @@ #####-------------------------------------------------------------------------- def get_mask(a): """Returns the mask of a, if any, or nomask. -To get a full array of booleans of the same shape as a, use getmaskarray.""" +To get a full array of booleans of the same shape as a, use getmaskarray. + """ return getattr(a, '_mask', nomask) getmask = get_mask def getmaskarray(a): - """Returns the mask of a, if any, or an array of the shape of a, full of False. + """Returns the mask of a, if any, or a boolean array of the shape of a, full of False. """ m = getmask(a) if m is nomask: @@ -621,17 +646,16 @@ return False # def make_mask(m, copy=False, shrink=True, flag=None): - """make_mask(m, copy=0, shrink=0) -Returns m as a mask, creating a copy if necessary or requested. + """Returns m as a mask, creating a copy if necessary or requested. The function can accept any sequence of integers or nomask. Does not check that contents must be 0s and 1s. -:Parameters: - m : ndarray +*Parameters*: + m : {ndarray} Potential mask. - copy : boolean *[False]* - Whether to return a copy of m. - shrink : boolean *[True]* + copy : {boolean} + Whether to return a copy of m (True) or m itself (False). + shrink : {boolean} Whether to shrink m to nomask if all its values are False. """ if flag is not None: @@ -660,8 +684,8 @@ def make_mask_none(s): """Returns a mask of shape s, filled with False. -:Parameters: - s : tuple +*Parameters*: + s : {tuple} A tuple indicating the shape of the final mask. """ result = numeric.zeros(s, dtype=MaskType) @@ -672,14 +696,14 @@ The masks are combined with the *logical_or* operator, treating nomask as False. The result may equal m1 or m2 if the other is nomask. -:Parameters: - m1 : ndarray +*Parameters*: + m1 : {ndarray} First mask. - m2 : ndarray + m2 : {ndarray} Second mask - copy : boolean *[False]* + copy : {boolean} Whether to return a copy. - shrink : boolean *[True]* + shrink : {boolean} Whether to shrink m to nomask if all its values are False. """ if m1 is nomask: @@ -697,13 +721,13 @@ """Returns a as an array masked where condition is true. Masked values of a or condition are kept. -:Parameters: - condition : ndarray +*Parameters*: + condition : {ndarray} Masking condition. - a : ndarray + a : {ndarray} Array to mask. - copy : boolean *[True]* - Whether to return a copy of a. + copy : {boolean} + Whether to return a copy of a (True) or modify a in place. """ cond = filled(condition,1) a = narray(a, copy=copy, subok=True) @@ -748,9 +772,12 @@ # return array(d, mask=m, copy=copy) def masked_inside(x, v1, v2, copy=True): - """Shortcut to masked_where, where condition is True for x inside -the interval [v1,v2] (v1 <= x <= v2). + """Shortcut to masked_where, where condition is True for x inside the interval +[v1,v2] (v1 <= x <= v2). The boundaries v1 and v2 can be given in either order. + +*Note*: + The array x is prefilled with its filling value. """ if v2 < v1: (v1, v2) = (v2, v1) @@ -759,9 +786,12 @@ return masked_where(condition, x, copy=copy) def masked_outside(x, v1, v2, copy=True): - """Shortcut to masked_where, where condition is True for x outside -the interval [v1,v2] (x < v1)|(x > v2). + """Shortcut to masked_where, where condition is True for x outside the interval +[v1,v2] (x < v1)|(x > v2). The boundaries v1 and v2 can be given in either order. + +*Note*: + The array x is prefilled with its filling value. """ if v2 < v1: (v1, v2) = (v2, v1) @@ -772,9 +802,12 @@ # def masked_object(x, value, copy=True): """Masks the array x where the data are exactly equal to value. -This function is suitable only for object arrays: for floating point, -please use masked_values instead. -The mask is set to `nomask` if posible. + +This function is suitable only for object arrays: for floating point, please use +``masked_values`` instead. + +*Notes*: + The mask is set to `nomask` if posible. """ if isMaskedArray(x): condition = umath.equal(x._data, value) @@ -788,19 +821,19 @@ def masked_values(x, value, rtol=1.e-5, atol=1.e-8, copy=True): """Masks the array x where the data are approximately equal to value (abs(x - value) <= atol+rtol*abs(value)). -Suitable only for floating points. For integers, please use masked_equal. +Suitable only for floating points. For integers, please use ``masked_equal``. The mask is set to nomask if posible. -:Parameters: - x : ndarray +*Parameters*: + x : {ndarray} Array to fill. - value : float + value : {float} Masking value. - rtol : float *[1e-5]* + rtol : {float} Tolerance parameter. - atol : float, *[1e-8]* + atol : {float}, *[1e-8]* Tolerance parameter. - copy : boolean *[True]* + copy : {boolean} Whether to return a copy of x. """ abs = umath.absolute @@ -870,16 +903,16 @@ #............................................................................... class _arraymethod(object): """Defines a wrapper for basic array methods. -Upon call, returns a masked array, where the new _data array is the output -of the corresponding method called on the original _data. +Upon call, returns a masked array, where the new _data array is the output of +the corresponding method called on the original _data. -If onmask is True, the new mask is the output of the method called on the initial mask. -If onmask is False, the new mask is just a reference to the initial mask. +If onmask is True, the new mask is the output of the method called on the initial +mask. Otherwise, the new mask is just a reference to the initial mask. :IVariables: _name : String Name of the function to apply on data. - _onmask : Boolean *[True]* + _onmask : {boolean} *[True]* Whether the mask must be processed also (True) or left alone (False). obj : Object The object calling the arraymethod @@ -920,7 +953,7 @@ return result #.......................................................... -class flatiter(object): +class FlatIter(object): "Defines an interator." def __init__(self, ma): self.ma = ma @@ -954,28 +987,29 @@ x = MaskedArray(data, mask=nomask, dtype=None, copy=True, fill_value=None, mask = nomask, fill_value=None, shrink=True) -:Parameters: - data : var +*Parameters*: + data : {var} Input data. - mask : sequence *[nomask]* + mask : {nomask, sequence} Mask. Must be convertible to an array of booleans with the same shape as data: True indicates a masked (eg., invalid) data. - dtype : dtype *[None]* + dtype : {dtype} Data type of the output. If None, the type of the data argument is used. If dtype is not None and different from data.dtype, a copy is performed. - copy : boolean *[False]* + copy : {boolean} Whether to copy the input data (True), or to use a reference instead. - fill_value : var *[None]* + Note: data are NOT copied by default. + fill_value : {var} Value used to fill in the masked values when necessary. If None, a default based on the datatype is used. - keep_mask : boolean *[True]* + keep_mask : {True, boolean} Whether to combine mask with the mask of the input data, if any (True), or to use only mask for the output (False). - hard_mask : boolean *[False]* + hard_mask : {False, boolean} Whether to use a hard mask or not. With a hard mask, masked values cannot be unmasked. - subok : boolean *[True]* + subok : {True, boolean} Whether to return a subclass of MaskedArray (if possible) or a plain MaskedArray. """ @@ -988,9 +1022,8 @@ def __new__(cls, data=None, mask=nomask, dtype=None, copy=False, fill_value=None, keep_mask=True, hard_mask=False, flag=None, subok=True, **options): - """array(data, dtype=None, copy=True, mask=nomask, fill_value=None) - -If `data` is already a ndarray, its dtype becomes the default value of dtype. + """Creates a new masked array from scratch. + Note: you can also create an array with the .view(MaskedArray) method... """ if flag is not None: warnings.warn("The flag 'flag' is now called 'shrink'!", @@ -1127,7 +1160,7 @@ #............................................. def __getitem__(self, indx): """x.__getitem__(y) <==> x[y] -Returns the item described by i. Not a copy as in previous versions. +Returns the item described by i, as a masked array. """ # This test is useful, but we should keep things light... # if getmask(indx) is not nomask: @@ -1200,14 +1233,14 @@ #............................................ def __getslice__(self, i, j): """x.__getslice__(i, j) <==> x[i:j] -Returns the slice described by i, j. +Returns the slice described by (i, j). The use of negative indices is not supported.""" return self.__getitem__(slice(i,j)) #........................ def __setslice__(self, i, j, value): """x.__setslice__(i, j, value) <==> x[i:j]=value -Sets a slice i:j to `value`. -If `value` is masked, masks those locations.""" +Sets the slice (i,j) of a to value. If value is masked, masks those locations. + """ self.__setitem__(slice(i,j), value) #............................................ def __setmask__(self, mask, copy=False): @@ -1262,27 +1295,27 @@ #............................................ def _get_data(self): - "Returns the current data (as a view of the original underlying data)>" + "Returns the current data, as a view of the original underlying data." return self.view(self._baseclass) _data = property(fget=_get_data) data = property(fget=_get_data) def raw_data(self): - """Returns the `_data` part of the MaskedArray. -You should really use `data` instead...""" + """Returns the _data part of the MaskedArray. +DEPRECATED: You should really use ``.data`` instead...""" return self._data #............................................ def _get_flat(self): "Returns a flat iterator." - return flatiter(self) + return FlatIter(self) # def _set_flat (self, value): "Sets a flattened version of self to value." - "x.flat = value" y = self.ravel() y[:] = value # - flat = property(fget=_get_flat, fset=_set_flat, doc="Flat version") + flat = property(fget=_get_flat, fset=_set_flat, + doc="Flat version of the array.") #............................................ def get_fill_value(self): "Returns the filling value." @@ -1292,27 +1325,30 @@ def set_fill_value(self, value=None): """Sets the filling value to value. -If None, uses a default based on the data type.""" +If value is None, uses a default based on the data type.""" if value is None: value = default_fill_value(self) self._fill_value = value fill_value = property(fget=get_fill_value, fset=set_fill_value, - doc="Filling value") + doc="Filling value.") def filled(self, fill_value=None): """Returns a copy of self._data, where masked values are filled with - fill_value. If fill_value is None, self.fill_value is used instead. - Subclassing is preserved. - Note : the result is NOT a MaskedArray ! +fill_value. -Examples --------- ->>> x = array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) ->>> x.filled() -array([1,2,-999,4,-999]) ->>> type(x.filled()) - +If fill_value is None, self.fill_value is used instead. + +*Note*: + + Subclassing is preserved + + The result is NOT a MaskedArray ! + +*Examples*: + >>> x = array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) + >>> x.filled() + array([1,2,-999,4,-999]) + >>> type(x.filled()) + """ m = self._mask if m is nomask or not m.any(): @@ -1353,9 +1389,7 @@ #............................................ def __str__(self): - """x.__str__() <==> str(x) -Calculates the string representation, using masked for fill if it is enabled. -Otherwise, fills with fill value. + """Calculates the string representation. """ if masked_print_option.enabled(): f = masked_print_option @@ -1381,9 +1415,7 @@ return str(res) def __repr__(self): - """x.__repr__() <==> repr(x) -Calculates the repr representation, using masked for fill if it is enabled. -Otherwise fill with fill value. + """Calculates the repr representation. """ with_mask = """\ masked_%(name)s(data = @@ -1497,10 +1529,18 @@ return int(self.item()) #............................................ def count(self, axis=None): - """Counts the non-masked elements of the array along a given axis, -and returns a masked array where the mask is True where all data are masked. -If axis is None, counts all the non-masked elements, and returns either a -scalar or the masked singleton.""" + """Counts the non-masked elements of the array along the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to count the non-masked elements. If not given, all the + non masked elements are counted. + +*Returns*: + A masked array where the mask is True where all data are masked. + If axis is None, returns either a scalar ot the masked singleton if all values + are masked. + """ m = self._mask s = self.shape ls = len(s) @@ -1538,8 +1578,13 @@ # def reshape (self, *s): """Reshapes the array to shape s. - Returns a new masked array. - If you want to modify the shape in place, please use a.shape = s""" + +*Returns*: + A new masked array. + +*Notes: + If you want to modify the shape in place, please use ``a.shape = s`` + """ result = self._data.reshape(*s).view(type(self)) result.__dict__.update(self.__dict__) if result._mask is not nomask: @@ -1548,9 +1593,12 @@ return result # def resize(self, newshape, refcheck=True, order=False): - """Attempts to modify size and shape of self inplace. - The array must own its own memory and not be referenced by other arrays. - Returns None. + """Attempts to modify the size and the shape of the array in place. + +The array must own its own memory and not be referenced by other arrays. + +*Returns*: + None. """ try: self._data.resize(newshape, refcheck, order) @@ -1564,10 +1612,11 @@ # def put(self, indices, values, mode='raise'): """Sets storage-indexed locations to corresponding values. - a.put(values, indices, mode) sets a.flat[n] = values[n] for each n in indices. - If values is shorter than indices then it will repeat. - If values has some masked values, the initial mask is updated in consequence, - else the corresponding values are unmasked. + +a.put(values, indices, mode) sets a.flat[n] = values[n] for each n in indices. +If values is shorter than indices then it will repeat. +If values has some masked values, the initial mask is updated in consequence, +else the corresponding values are unmasked. """ m = self._mask # Hard mask: Get rid of the values/indices that fall on masked data @@ -1597,42 +1646,86 @@ return (self.ctypes.data, self._mask.ctypes.data) #............................................ def all(self, axis=None, out=None): - """a.all(axis) returns True if all entries along the axis are True. - Returns False otherwise. If axis is None, uses the flatten array. - Masked values are considered as True during computation. - Outputs a masked array, where the mask is True if all data are masked along the axis. - Note: the out argument is not really operational... + """Returns True if all entries along the given axis are True, False otherwise. +Masked values are considered as True during computation. + +*Parameters* + axis : {integer}, optional + Axis along which the operation is performed. + If None, the operation is performed on a flatten array + out : {MaskedArray}, optional + Alternate optional output. + If not None, out should be a valid MaskedArray of the same shape as the + output of self._data.all(axis). + +*Returns* + A masked array, where the mask is True if all data along the axis are masked. + +*Notes* + An exception is raised if ``out`` is not None and not of the same type as self. """ - d = self.filled(True).all(axis=axis, out=out).view(type(self)) - if d.ndim > 0: - d.__setmask__(self._mask.all(axis)) - return d + if out is None: + d = self.filled(True).all(axis=axis).view(type(self)) + if d.ndim > 0: + d.__setmask__(self._mask.all(axis)) + return d + elif type(out) is not type(self): + raise TypeError("The external array should have a type %s (got %s instead)" %\ + (type(self), type(out))) + self.filled(True).all(axis=axis, out=out) + if out.ndim: + out.__setmask__(self._mask.all(axis)) + return out + def any(self, axis=None, out=None): - """a.any(axis) returns True if some or all entries along the axis are True. - Returns False otherwise. If axis is None, uses the flatten array. - Masked data are considered as False during computation. - Outputs a masked array, where the mask is True if all data are masked along the axis. - Note: the out argument is not really operational... + """Returns True if at least one entry along the given axis is True. + +Returns False if all entries are False. +Masked values are considered as True during computation. + +*Parameters* + axis : {integer}, optional + Axis along which the operation is performed. + If None, the operation is performed on a flatten array + out : {MaskedArray}, optional + Alternate optional output. + If not None, out should be a valid MaskedArray of the same shape as the + output of self._data.all(axis). + +*Returns* + A masked array, where the mask is True if all data along the axis are masked. + +*Notes* + An exception is raised if ``out`` is not None and not of the same type as self. """ - d = self.filled(False).any(axis=axis, out=out).view(type(self)) - if d.ndim > 0: - d.__setmask__(self._mask.all(axis)) - return d + if out is None: + d = self.filled(True).any(axis=axis).view(type(self)) + if d.ndim > 0: + d.__setmask__(self._mask.all(axis)) + return d + elif type(out) is not type(self): + raise TypeError("The external array should have a type %s (got %s instead)" %\ + (type(self), type(out))) + self.filled(True).any(axis=axis, out=out) + if out.ndim: + out.__setmask__(self._mask.all(axis)) + return out + def nonzero(self): - """a.nonzero() returns the indices of the elements of a that are not - zero nor masked, as a tuple of arrays. + """Returns the indices of the elements of a that are not zero nor masked, +as a tuple of arrays. - There are as many tuples as dimensions of a, each tuple contains the indices - of the non-zero elements in that dimension. The corresponding non-zero values - can be obtained with - a[a.nonzero()]. +There are as many tuples as dimensions of a, each tuple contains the indices of +the non-zero elements in that dimension. The corresponding non-zero values can +be obtained with ``a[a.nonzero()]``. - To group the indices by element, rather than dimension, use - transpose(a.nonzero()) - instead. The result of this is always a 2d array, with a row for - each non-zero element.""" +To group the indices by element, rather than dimension, use instead: +``transpose(a.nonzero())``. + +The result of this is always a 2d array, with a row for each non-zero element. + """ return narray(self.filled(0), copy=False).nonzero() #............................................ def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None): @@ -1650,16 +1743,17 @@ return D.astype(dtype).filled(0).sum(axis=None) #............................................ def sum(self, axis=None, dtype=None): - """a.sum(axis=None, dtype=None) - Sums the array a over the given axis. - Masked elements are set to 0. + """Sums the array over the given axis. + +Masked elements are set to 0 internally. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. """ if self._mask is nomask: mask = nomask @@ -1673,32 +1767,34 @@ return result def cumsum(self, axis=None, dtype=None): - """a.cumprod(axis=None, dtype=None) - Returns the cumulative sum of the elements of a along the given axis. - Masked values are set to 0. + """Returns the cumulative sum of the elements of the array along the given axis. + +Masked values are set to 0 internally. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. """ result = self.filled(0).cumsum(axis=axis, dtype=dtype).view(type(self)) result.__setmask__(self.mask) return result def prod(self, axis=None, dtype=None): - """a.prod(axis=None, dtype=None) - Returns the product of the elements of a along the given axis. - Masked elements are set to 1. + """Returns the product of the elements of the array along the given axis. + +Masked elements are set to 1 internally. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. """ if self._mask is nomask: mask = nomask @@ -1713,34 +1809,34 @@ product = prod def cumprod(self, axis=None, dtype=None): - """a.cumprod(axis=None, dtype=None) - Returns the cumulative product of the elements of a along the given axis. - Masked values are set to 1. + """Returns the cumulative product of the elements of the array along the given axis. + +Masked values are set to 1 internally. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. """ result = self.filled(1).cumprod(axis=axis, dtype=dtype).view(type(self)) result.__setmask__(self.mask) return result def mean(self, axis=None, dtype=None): - """a.mean(axis=None, dtype=None) + """Averages the array over the given axis. Equivalent to - Averages the array over the given axis. Equivalent to - - a.sum(axis, dtype) / size(a, axis). + a.sum(axis, dtype) / a.size(axis). -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. """ if self._mask is nomask: return super(MaskedArray, self).mean(axis=axis, dtype=dtype) @@ -1750,15 +1846,15 @@ return dsum*1./cnt def anom(self, axis=None, dtype=None): - """a.anom(axis=None, dtype=None) - Returns the anomalies, or deviation from the average. + """Returns the anomalies (deviations from the average) along the given axis. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. """ m = self.mean(axis, dtype) if not axis: @@ -1767,23 +1863,22 @@ return (self - expand_dims(m,axis)) def var(self, axis=None, dtype=None): - """a.var(axis=None, dtype=None) -Returns the variance, a measure of the spread of a distribution. + """Returns the variance, a measure of the spread of a distribution. + The variance is the average of the squared deviations from the mean, i.e. var = mean((x - x.mean())**2). -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. - + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. -Notes ------ -The value returned is a biased estimate of the true variance. -For the more standard unbiased estimate, use varu. +*Notes*: + The value returned is a biased estimate of the true variance. + For the (more standard) unbiased estimate, use varu. """ if self._mask is nomask: # TODO: Do we keep super, or var _data and take a view ? @@ -1799,24 +1894,22 @@ return dvar def std(self, axis=None, dtype=None): - """a.std(axis=None, dtype=None) -Returns the standard deviation, a measure of the spread of a distribution. + """Returns the standard deviation, a measure of the spread of a distribution. The standard deviation is the square root of the average of the squared deviations from the mean, i.e. std = sqrt(mean((x - x.mean())**2)). -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - dtype : dtype *[None]* - Datatype for the intermediary computation. + dtype : {dtype}, optional + Datatype for the intermediary computation. + If not given, the current dtype is used instead. - -Notes ------ -The value returned is a biased estimate of the true standard deviation. -For the more standard unbiased estimate, use stdu. +*Notes*: + The value returned is a biased estimate of the true standard deviation. + For the more standard unbiased estimate, use stdu. """ dvar = self.var(axis,dtype) if axis is not None or dvar is not masked: @@ -1826,30 +1919,30 @@ #............................................ def argsort(self, axis=None, fill_value=None, kind='quicksort', order=None): - """Returns a ndarray of indices that sort 'a' along the specified axis. + """Returns a ndarray of indices that sort the array along the specified axis. Masked values are filled beforehand to fill_value. Returns a numpy array. -:Parameters: - axis : integer *[None]* - Axis to be indirectly sorted. - fill_value : var *[None]* - Value used to fill in the masked values. - If None, use self.fill_value instead. - kind : String *['quicksort']* +*Parameters*: + axis : {integer}, optional + Axis to be indirectly sorted. + If not given, uses a flatten version of the array. + fill_value : {var} + Value used to fill in the masked values. + If not given, self.fill_value is used instead. + kind : {string} Sorting algorithm (default 'quicksort') Possible values: 'quicksort', 'mergesort', or 'heapsort' -Notes: ------- - This method executes an indirect sort along the given axis using the - algorithm specified by the kind keyword. It returns an array of indices of - the same shape as 'a' that index data along the given axis in sorted order. +*Notes*: + This method executes an indirect sort along the given axis using the algorithm + specified by the kind keyword. It returns an array of indices of the same shape + as 'a' that index data along the given axis in sorted order. - The various sorts are characterized by average speed, worst case - performance, need for work space, and whether they are stable. A stable - sort keeps items with the same key in the same relative order. The three - available algorithms have the following properties: + The various sorts are characterized by average speed, worst case performance + need for work space, and whether they are stable. A stable sort keeps items + with the same key in the same relative order. The three available algorithms + have the following properties: |------------------------------------------------------| | kind | speed | worst case | work space | stable| @@ -1870,15 +1963,17 @@ #........................ def argmin(self, axis=None, fill_value=None): """Returns a ndarray of indices for the minimum values of a along the - specified axis. - Masked values are treated as if they had the value fill_value. +specified axis. -:Parameters: - axis : integer *[None]* - Axis to be indirectly sorted. - fill_value : var *[None]* +Masked values are treated as if they had the value fill_value. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional Value used to fill in the masked values. - If None, use the the output of minimum_fill_value(). + If None, the output of minimum_fill_value(self._data) is used. """ if fill_value is None: fill_value = minimum_fill_value(self) @@ -1887,15 +1982,17 @@ #........................ def argmax(self, axis=None, fill_value=None): """Returns the array of indices for the maximum values of `a` along the - specified axis. - Masked values are treated as if they had the value `fill_value`. +specified axis. -:Parameters: - axis : integer *[None]* - Axis to be indirectly sorted. - fill_value : var *[None]* +Masked values are treated as if they had the value fill_value. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional Value used to fill in the masked values. - If None, use the the output of maximum_fill_value(). + If None, the output of maximum_fill_value(self._data) is used. """ if fill_value is None: fill_value = maximum_fill_value(self._data) @@ -1906,37 +2003,36 @@ endwith=True, fill_value=None): """Sort a along the given axis. -:Parameters: - axis : integer *[-1]* +*Parameters*: + axis : {integer} Axis to be indirectly sorted. - kind : String *['quicksort']* + kind : {string} Sorting algorithm (default 'quicksort') Possible values: 'quicksort', 'mergesort', or 'heapsort'. - order : var *[None]* + order : {var} If a has fields defined, then the order keyword can be the field name to sort on or a list (or tuple) of field names to indicate the order that fields should be used to define the sort. - fill_value : var *[None]* + fill_value : {var} Value used to fill in the masked values. If None, use the the output of minimum_fill_value(). - endwith : boolean *[True]* + endwith : {boolean} Whether missing values (if any) should be forced in the upper indices (at the end of the array) (True) or lower indices (at the beginning). -:Returns: +*Returns*: When used as method, returns None. When used as a function, returns an array. -Notes ------ +*Notes*: This method sorts 'a' in place along the given axis using the algorithm specified by the kind keyword. - The various sorts may characterized by average speed, worst case - performance, need for work space, and whether they are stable. A stable - sort keeps items with the same key in the same relative order and is most - useful when used with argsort where the key might differ from the items - being sorted. The three available algorithms have the following properties: + The various sorts may characterized by average speed, worst case performance + need for work space, and whether they are stable. A stable sort keeps items + with the same key in the same relative order and is most useful when used w/ + argsort where the key might differ from the items being sorted. + The three available algorithms have the following properties: |------------------------------------------------------| | kind | speed | worst case | work space | stable| @@ -1969,13 +2065,14 @@ #............................................ def min(self, axis=None, fill_value=None): """Returns the minimum of a along the given axis. - Masked values are filled with fill_value. -:Parameters: - axis : integer *[None]* +Masked values are filled with fill_value. + +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - fill_value : var *[None]* + fill_value : {var}, optional Value used to fill in the masked values. If None, use the the output of minimum_fill_value(). """ @@ -2001,13 +2098,14 @@ #........................ def max(self, axis=None, fill_value=None): """Returns the maximum/a along the given axis. - Masked values are filled with fill_value. + +Masked values are filled with fill_value. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - fill_value : var *[None]* + fill_value : {var}, optional Value used to fill in the masked values. If None, use the the output of maximum_fill_value(). """ @@ -2034,11 +2132,11 @@ def ptp(self, axis=None, fill_value=None): """Returns the visible data range (max-min) along the given axis. -:Parameters: - axis : integer *[None]* +*Parameters*: + axis : {integer}, optional Axis along which to perform the operation. If None, applies to a flattened version of the array. - fill_value : var *[None]* + fill_value : {var}, optional Value used to fill in the masked values. If None, the maximum uses the maximum default, the minimum uses the minimum default. @@ -2062,10 +2160,11 @@ #-------------------------------------------- def tolist(self, fill_value=None): """Copies the data portion of the array to a hierarchical python list and - returns that list. Data items are converted to the nearest compatible Python - type. - Masked values are converted to fill_value. If fill_value is None, the - corresponding entries in the output list will be None. +returns that list. + +Data items are converted to the nearest compatible Python type. +Masked values are converted to fill_value. If fill_value is None, the corresponding +entries in the output list will be ``None``. """ if fill_value is not None: return self.filled(fill_value).tolist() @@ -2083,21 +2182,17 @@ for i in idx[:-1]: tmp = tmp[i] tmp[idx[-1]] = None - return result - - + return result #........................ def tostring(self, fill_value=None, order='C'): - """a.tostring(order='C', fill_value=None) - - Returns a copy of array data as a Python string containing the - raw bytes in the array. + """Returns a copy of array data as a Python string containing the raw +bytes in the array. -:Parameters: - fill_value : var *[None]* +*Parameters*: + fill_value : {var}, optional Value used to fill in the masked values. If None, uses self.fill_value instead. - order : string *['C']* + order : {string} Order of the data item in the copy {"C","F","A"}. "C" -- C order (row major) "Fortran" -- Fortran order (column major) @@ -2121,7 +2216,7 @@ # def __setstate__(self, state): """Restores the internal state of the masked array, for pickling purposes. - `state` is typically the output of the ``__getstate__`` output, and is a 5-tuple: +``state`` is typically the output of the ``__getstate__`` output, and is a 5-tuple: - class name - a tuple giving the shape of the data @@ -2419,7 +2514,7 @@ return x.compressed() def concatenate(arrays, axis=0): - "Concatenates the arrays along the given axis" + "Concatenates the arrays along the given axis." d = numpy.concatenate([getdata(a) for a in arrays], axis) rcls = get_masked_subclass(*arrays) data = d.view(rcls) @@ -2438,12 +2533,12 @@ return data def count(a, axis = None): - "Count of the non-masked elements in a, or along a certain axis." return masked_array(a, copy=False).count(axis) +count.__doc__ = MaskedArray.count.__doc__ def expand_dims(x,axis): - """Expands the shape of a by including newaxis before given axis.""" + "Expands the shape of the array by including a new axis before the given one." result = n_expand_dims(x,axis) if isinstance(x, MaskedArray): new_shape = result.shape @@ -2455,7 +2550,7 @@ #...................................... def left_shift (a, n): - "Left shift n bits" + "Left shift n bits." m = getmask(a) if m is nomask: d = umath.left_shift(filled(a), n) @@ -2465,7 +2560,7 @@ return masked_array(d, mask=m) def right_shift (a, n): - "Right shift n bits" + "Right shift n bits." m = getmask(a) if m is nomask: d = umath.right_shift(filled(a), n) @@ -2477,7 +2572,7 @@ #...................................... def put(a, indices, values, mode='raise'): """Sets storage-indexed locations to corresponding values. - Values and indices are filled if necessary.""" +Values and indices are filled if necessary.""" # We can't use 'frommethod', the order of arguments is different try: return a.put(indices, values, mode=mode) @@ -2485,9 +2580,10 @@ return narray(a, copy=False).put(indices, values, mode=mode) def putmask(a, mask, values): #, mode='raise'): - """`putmask(a, mask, v)` results in `a = v` for all places where `mask` is true. -If `v` is shorter than `mask`, it will be repeated as necessary. -In particular `v` can be a scalar or length 1 array.""" + """Sets a.flat[n] = values[n] for each n where mask.flat[n] is true. + +If values is not the same size of a and mask then it will repeat as necessary. +This gives different behavior than a[mask] = values.""" # We can't use 'frommethod', the order of arguments is different try: return a.putmask(values, mask) @@ -2495,14 +2591,17 @@ return narray(a, copy=False).putmask(values, mask) def transpose(a,axes=None): - """Returns a view of the array with dimensions permuted according to axes. -If `axes` is None (default), returns array with dimensions reversed. + """Returns a view of the array with dimensions permuted according to axes, +as a masked array. + +If ``axes`` is None (default), the output view has reversed dimensions compared +to the original. """ #We can't use 'frommethod', as 'transpose' doesn't take keywords try: return a.transpose(axes) except AttributeError: - return narray(a, copy=False).transpose(axes) + return narray(a, copy=False).transpose(axes).view(MaskedArray) def reshape(a, new_shape): """Changes the shape of the array a to new_shape.""" @@ -2510,13 +2609,14 @@ try: return a.reshape(new_shape) except AttributeError: - return narray(a, copy=False).reshape(new_shape) + return narray(a, copy=False).reshape(new_shape).view(MaskedArray) def resize(x, new_shape): - """resize(a,new_shape) returns a new array with the specified shape. - The total size of the original array can be any size. - The new array is filled with repeated copies of a. If a was masked, the new - array will be masked, and the new mask will be a repetition of the old one. + """Returns a new array with the specified shape. + +The total size of the original array can be any size. +The new array is filled with repeated copies of a. If a was masked, the new array +will be masked, and the new mask will be a repetition of the old one. """ # We can't use _frommethods here, as N.resize is notoriously whiny. m = getmask(x) @@ -2530,20 +2630,19 @@ #................................................ def rank(obj): - """Gets the rank of sequence a (the number of dimensions, not a matrix rank) -The rank of a scalar is zero.""" + "maskedarray version of the numpy function." return fromnumeric.rank(getdata(obj)) +rank.__doc__ = numpy.rank.__doc__ # def shape(obj): - """Returns the shape of `a` (as a function call which also works on nested sequences). - """ + "maskedarray version of the numpy function." return fromnumeric.shape(getdata(obj)) +shape.__doc__ = numpy.shape.__doc__ # def size(obj, axis=None): - """Returns the number of elements in the array along the given axis, -or in the sequence if `axis` is None. - """ + "maskedarray version of the numpy function." return fromnumeric.size(getdata(obj), axis) +size.__doc__ = numpy.size.__doc__ #................................................ #####-------------------------------------------------------------------------- @@ -2551,16 +2650,23 @@ #####-------------------------------------------------------------------------- def where (condition, x=None, y=None): """where(condition | x, y) - Returns a (subclass of) masked array, shaped like condition, where - the elements are x when condition is True, and y otherwise. - condition must be convertible to an integer array. - If neither x nor y are given, returns a tuple of indices where condition is - True (a la condition.nonzero()). + +Returns a (subclass of) masked array, shaped like condition, where the elements +are x when condition is True, and y otherwise. If neither x nor y are given, +returns a tuple of indices where condition is True (a la condition.nonzero()). + +*Parameters*: + condition : {var} + The condition to meet. Must be convertible to an integer array. + x : {var}, optional + Values of the output when the condition is met + y : {var}, optional + Values of the output when the condition is not met. """ if x is None and y is None: return filled(condition, 0).nonzero() elif x is None or y is None: - raise ValueError, "Either bioth or neither x and y should be given." + raise ValueError, "Either both or neither x and y should be given." # Get the condition ............... fc = filled(condition, 0).astype(bool_) notfc = numpy.logical_not(fc) @@ -2628,28 +2734,36 @@ return masked_array(d, mask=m) def round_(a, decimals=0, out=None): - """Returns a copy of a rounded to 'decimals' places. + """Returns a copy of a, rounded to 'decimals' places. + +When 'decimals' is negative, it specifies the number of positions to the left of +the decimal point. The real and imaginary parts of complex numbers are rounded +separately. Nothing is done if the array is not of float type and 'decimals' is +greater than or equal to 0. -:Parameters: - decimals : integer *[0]* +*Parameters*: + decimals : {integer} Number of decimals to round to. May be negative. - out : ndarray - Existing array to use for output (default copy of a). - -Notes ------ - Rounds to the specified number of decimals. When 'decimals' is negative, - it specifies the number of positions to the left of the decimal point. - The real and imaginary parts of complex numbers are rounded separately. - Nothing is done if the array is not of float type and 'decimals' is greater - than or equal to 0.""" - result = fromnumeric.round_(getdata(a), decimals, out) - if isinstance(a,MaskedArray): - result = result.view(type(a)) - result._mask = a._mask + out : {ndarray} + Existing array to use for output. + If not given, returns a default copy of a. + +*Notes*: + If out is given and does not have a mask attribute, the mask of a is lost! + """ + if out is None: + result = fromnumeric.round_(getdata(a), decimals, out) + if isinstance(a,MaskedArray): + result = result.view(type(a)) + result._mask = a._mask + else: + result = result.view(MaskedArray) + return result else: - result = result.view(MaskedArray) - return result + fromnumeric.round_(getdata(a), decimals, out) + if hasattr(out, '_mask'): + out._mask = getmask(a) + return out def arange(stop, start=None, step=1, dtype=None): "maskedarray version of the numpy function." @@ -2665,7 +2779,8 @@ if len(fb.shape) == 0: fb.shape = (1,) return numpy.inner(fa, fb).view(MaskedArray) -inner.__doc__ = numpy.inner.__doc__ + "\nMasked values are replaced by 0." +inner.__doc__ = numpy.inner.__doc__ +inner.__doc__ += "\n*Notes*:\n Masked values are replaced by 0." innerproduct = inner def outer(a, b): @@ -2681,12 +2796,13 @@ mb = getmaskarray(b) m = make_mask(1-numeric.outer(1-ma, 1-mb), copy=0) return masked_array(d, mask=m) -outer.__doc__ = numpy.outer.__doc__ + "\nMasked values are replaced by 0." +outer.__doc__ = numpy.outer.__doc__ +outer.__doc__ += "\n*Notes*:\n Masked values are replaced by 0." outerproduct = outer def allequal (a, b, fill_value=True): """Returns True if all entries of a and b are equal, using fill_value - as a truth value where either or both are masked. +as a truth value where either or both are masked. """ m = mask_or(getmask(a), getmask(b)) if m is nomask: @@ -2722,7 +2838,8 @@ #.............................................................................. def asarray(a, dtype=None): """asarray(data, dtype) = array(data, dtype, copy=0, subok=0) -Returns a as a MaskedArray object. +Returns a as a MaskedArray object of the given dtype. +If dtype is not given or None, is is set to the dtype of a. No copy is performed if a is already an array. Subclasses are converted to the base class MaskedArray. """ @@ -2731,6 +2848,7 @@ def asanyarray(a, dtype=None): """asanyarray(data, dtype) = array(data, dtype, copy=0, subok=1) Returns a as an masked array. +If dtype is not given or None, is is set to the dtype of a. No copy is performed if a is already an array. Subclasses are conserved. """ @@ -2814,4 +2932,6 @@ mmyl = array(yl, mask=masky, shrink=True) mmzl = array(zl, mask=maskx, shrink=True) # - z = log(mmxl) \ No newline at end of file + z = empty(3,) + mmys.all(0, out=z) + \ No newline at end of file Modified: trunk/scipy/sandbox/maskedarray/extras.py =================================================================== --- trunk/scipy/sandbox/maskedarray/extras.py 2007-10-03 00:06:40 UTC (rev 3389) +++ trunk/scipy/sandbox/maskedarray/extras.py 2007-10-03 01:46:24 UTC (rev 3390) @@ -50,13 +50,24 @@ return False def count_masked(arr, axis=None): - """Counts the number of masked elements along the given axis.""" + """Counts the number of masked elements along the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to count. + If None (default), a flattened version of the array is used. + """ m = getmaskarray(arr) return m.sum(axis) def masked_all(shape, dtype=float_): """Returns an empty masked array of the given shape and dtype, - where all the data are masked.""" + where all the data are masked. + +*Parameters*: + dtype : {dtype}, optional + Data type of the output. + """ a = masked_array(numeric.empty(shape, dtype), mask=numeric.ones(shape, bool_)) return a @@ -72,11 +83,20 @@ #---- --- New methods --- #####-------------------------------------------------------------------------- def varu(a, axis=None, dtype=None): - """a.var(axis=None, dtype=None) - Returns an unbiased estimate of the variance. + """Returns an unbiased estimate of the variance. + i.e. var = sum((x - x.mean())**2)/(size(x,axis)-1) - Instead of dividing the sum of squared anomalies (SSA) by n, the number of - elements, the SSA is divided by n-1. +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + +*Notes*: + The value returned is an unbiased estimate of the true variance. + For the (less standard) biased estimate, use var. """ a = asarray(a) cnt = a.count(axis=axis) @@ -92,11 +112,21 @@ # fill_value=a._fill_value) def stdu(a, axis=None, dtype=None): - """a.var(axis=None, dtype=None) - Returns an unbiased estimate of the standard deviation. + """Returns an unbiased estimate of the standard deviation. + The standard deviation is the square root of the average of the squared + deviations from the mean, i.e. stdu = sqrt(varu(x)). - Instead of dividing the sum of squared anomalies (SSA) by n, the number of - elements, the SSA is divided by n-1. +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. + If not given, the current dtype is used instead. + +*Notes*: + The value returned is an unbiased estimate of the true standard deviation. + For the (less standard) biased estimate, use std. """ a = asarray(a) dvar = a.varu(axis,dtype) @@ -124,7 +154,7 @@ def getdoc(self): "Retrieves the __doc__ string from the function." return getattr(numpy, self._function).__doc__ +\ - "(The function is applied to both the _data and the mask, if any.)" + "*Notes*:\n (The function is applied to both the _data and the _mask, if any.)" def __call__(self, *args, **params): func = getattr(numpy, self._function) if len(args)==1: @@ -260,31 +290,21 @@ result.fill_value = core.default_fill_value(result) return result -def average (a, axis=None, weights=None, returned = 0): - """average(a, axis=None weights=None, returned=False) - - Averages the array over the given axis. If the axis is None, averages - over all dimensions of the array. Equivalent to a.mean(axis) - - If an integer axis is given, this equals: - a.sum(axis) * 1.0 / size(a, axis) - - If axis is None, this equals: - a.sum(axis) * 1.0 / a.size - - If weights are given, result is: - sum(a * weights,axis) / sum(weights,axis), - where the weights must have a's shape or be 1D with length the - size of a in the given axis. Integer weights are converted to - Float. Not specifying weights is equivalent to specifying - weights that are all 1. - - If 'returned' is True, return a tuple: the result and the sum of - the weights or count of values. The shape of these two results - will be the same. - - Returns masked values instead of ZeroDivisionError if appropriate. +def average (a, axis=None, weights=None, returned=False): + """Averages the array over the given axis. +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + weights : {sequence}, optional + Sequence of weights. + The weights must have the shape of a, or be 1D with length the size of a + along the given axis. + If no weights are given, weights are assumed to be 1. + returned : {boolean} + Flag indicating whether a tuple (result, sum of weights/counts) should be + returned as output (True), or just the result (False). """ a = asarray(a) mask = a.mask @@ -380,14 +400,17 @@ #.............................................................................. def compress_rowcols(x, axis=None): """Suppresses the rows and/or columns of a 2D array that contains masked values. + The suppression behavior is selected with the `axis`parameter. - If axis is None, rows and columns are suppressed. - If axis is 0, only rows are suppressed. - If axis is 1 or -1, only columns are suppressed. - Returns a *pure* ndarray. + +*Returns*: + compressed_array : a ndarray. """ x = asarray(x) - if x.ndim <> 2: + if x.ndim != 2: raise NotImplementedError, "compress2d works for 2D arrays only." m = getmask(x) # Nothing is masked: return x @@ -418,9 +441,9 @@ def mask_rowcols(a, axis=None): """Masks whole rows and/or columns of a 2D array that contain masked values. The masking behavior is selected with the `axis`parameter. - - If axis is None, rows and columns are suppressed. - - If axis is 0, only rows are suppressed. - - If axis is 1 or -1, only columns are suppressed. + - If axis is None, rows and columns are masked. + - If axis is 0, only rows are masked. + - If axis is 1 or -1, only columns are masked. Returns a *pure* ndarray. """ a = asarray(a) @@ -449,13 +472,18 @@ def dot(a,b, strict=False): """Returns the dot product of two 2D masked arrays a and b. - Like the generic numpy equivalent the product sum is over - the last dimension of a and the second-to-last dimension of b. + Like the generic numpy equivalent, the product sum is over the last dimension + of a and the second-to-last dimension of b. If strict is True, masked values are propagated: if a masked value appears in a row or column, the whole row or column is considered masked. - NB: The first argument is not conjugated. +*Parameters*: + strict : {boolean} + Whether masked data are propagated (True) or set to 0 for the computation. + +*Note*: + The first argument is not conjugated. """ #TODO: Works only with 2D arrays. There should be a way to get it to run with higher dimension if strict and (a.ndim == 2) and (b.ndim == 2): @@ -471,7 +499,23 @@ #............................................................................... def mediff1d(array, to_end=None, to_begin=None): - """Array difference with prefixed and/or appended value.""" + """Returns the differences between consecutive elements of an array, possibly with + prefixed and/or appended values. + +*Parameters*: + array : {array} + Input array, will be flattened before the difference is taken. + to_end : {number}, optional + If provided, this number will be tacked onto the end of the returned + differences. + to_begin : {number}, optional + If provided, this number will be taked onto the beginning of the + returned differences. + +*Returns*: + ed : {array} + The differences. Loosely, this will be (ary[1:] - ary[:-1]). + """ a = masked_array(array, copy=True) if a.ndim > 1: a.reshape((a.size,)) @@ -585,7 +629,7 @@ """Translates slice objects to concatenation along the first axis. For example: - >>> r_[array([1,2,3]), 0, 0, array([4,5,6])] + >>> mr_[array([1,2,3]), 0, 0, array([4,5,6])] array([1, 2, 3, 0, 0, 4, 5, 6]) """ def __init__(self): @@ -598,7 +642,7 @@ #####-------------------------------------------------------------------------- def flatnotmasked_edges(a): - """Finds the indices of the first and last not masked values in a 1D masked array. + """Finds the indices of the first and last not masked values in a 1D masked array. If all values are masked, returns None. """ m = getmask(a) Modified: trunk/scipy/sandbox/maskedarray/morestats.py =================================================================== --- trunk/scipy/sandbox/maskedarray/morestats.py 2007-10-03 00:06:40 UTC (rev 3389) +++ trunk/scipy/sandbox/maskedarray/morestats.py 2007-10-03 01:46:24 UTC (rev 3390) @@ -38,20 +38,21 @@ #####-------------------------------------------------------------------------- def hdquantiles(data, prob=list([.25,.5,.75]), axis=None, var=False,): """Computes quantile estimates with the Harrell-Davis method, where the estimates - are calculated as a weighted linear combination of order statistics. - If var=True, the variance of the estimate is also returned. - Depending on var, returns a (p,) array of quantiles or a (2,p) array of quantiles - and variances. +are calculated as a weighted linear combination of order statistics. -:Inputs: - data: ndarray +*Parameters* : + data: {ndarray} Data array. - prob: Sequence - List of quantiles to compute. - axis : integer *[None]* + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} Axis along which to compute the quantiles. If None, use a flattened array. - var : boolean *[False]* + var : {boolean} Whether to return the variance of the estimate. + +*Returns* + A (p,) array of quantiles (if ``var`` is False), or a (2,p) array of quantiles + and variances (if ``var`` is True), where ``p`` is the number of quantiles. :Note: The function is restricted to 2D arrays. @@ -101,12 +102,12 @@ def hdmedian(data, axis=-1, var=False): """Returns the Harrell-Davis estimate of the median along the given axis. -:Inputs: - data: ndarray +*Parameters* : + data: {ndarray} Data array. - axis : integer *[None]* + axis : {integer} Axis along which to compute the quantiles. If None, use a flattened array. - var : boolean *[False]* + var : {boolean} Whether to return the variance of the estimate. """ result = hdquantiles(data,[0.5], axis=axis, var=var) @@ -117,19 +118,16 @@ def hdquantiles_sd(data, prob=list([.25,.5,.75]), axis=None): """Computes the standard error of the Harrell-Davis quantile estimates by jackknife. -:Inputs: - data: ndarray + +*Parameters* : + data: {ndarray} Data array. - prob: Sequence - List of quantiles to compute. - axis : integer *[None]* + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} Axis along which to compute the quantiles. If None, use a flattened array. - var : boolean *[False]* - Whether to return the variance of the estimate. - stderr : boolean *[False]* - Whether to return the standard error of the estimate. -:Note: +*Note*: The function is restricted to 2D arrays. """ def _hdsd_1D(data,prob): @@ -172,18 +170,18 @@ def trimmed_mean_ci(data, proportiontocut=0.2, alpha=0.05, axis=None): """Returns the selected confidence interval of the trimmed mean along the - given axis. +given axis. -:Inputs: - data : sequence +*Parameters* : + data : {sequence} Input data. The data is transformed to a masked array - proportiontocut : float *[0.2]* + proportiontocut : {float} Proportion of the data to cut from each side of the data . As a result, (2*proportiontocut*n) values are actually trimmed. - alpha : float *[0.05]* - Confidence level of the intervals - axis : integer *[None]* - Axis along which to cut. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to cut. If None, uses a flattened version of the input. """ data = masked_array(data, copy=False) trimmed = trim_both(data, proportiontocut=proportiontocut, axis=axis) @@ -196,15 +194,15 @@ #.............................................................................. def mjci(data, prob=[0.25,0.5,0.75], axis=None): """Returns the Maritz-Jarrett estimators of the standard error of selected - experimental quantiles of the data. +experimental quantiles of the data. -:Input: - data : sequence - Input data. - prob : sequence *[0.25,0.5,0.75]* - Sequence of quantiles whose standard error must be estimated. - axis : integer *[None]* - Axis along which to compute the standard error. +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. """ def _mjci_1D(data, p): data = data.compressed() @@ -236,17 +234,17 @@ #.............................................................................. def mquantiles_cimj(data, prob=[0.25,0.50,0.75], alpha=0.05, axis=None): """Computes the alpha confidence interval for the selected quantiles of the - data, with Maritz-Jarrett estimators. +data, with Maritz-Jarrett estimators. -:Input: - data : sequence - Input data. - prob : sequence *[0.25,0.5,0.75]* - Sequence of quantiles whose standard error must be estimated. - alpha : float *[0.05]* - Confidence degree. - axis : integer *[None]* - Axis along which to compute the standard error. +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. """ alpha = min(alpha, 1-alpha) z = norm.ppf(1-alpha/2.) @@ -258,13 +256,16 @@ #............................................................................. def median_cihs(data, alpha=0.05, axis=None): """Computes the alpha-level confidence interval for the median of the data, - following the Hettmasperger-Sheather method. +following the Hettmasperger-Sheather method. -:Inputs: - data : sequence - Input data. Masked values are discarded. The input should be 1D only - alpha : float *[0.05]* - Confidence degree. +*Parameters* : + data : {sequence} + Input data. Masked values are discarded. The input should be 1D only, or + axis should be set to None. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. """ def _cihs_1D(data, alpha): data = numpy.sort(data.compressed()) @@ -294,17 +295,21 @@ #.............................................................................. def compare_medians_ms(group_1, group_2, axis=None): """Compares the medians from two independent groups along the given axis. - Returns an array of p values. - The comparison is performed using the McKean-Schrader estimate of the standard - error of the medians. + +The comparison is performed using the McKean-Schrader estimate of the standard +error of the medians. -:Inputs: - group_1 : sequence +*Parameters* : + group_1 : {sequence} First dataset. - group_2 : sequence + group_2 : {sequence} Second dataset. - axis : integer *[None]* + axis : {integer} Axis along which the medians are estimated. If None, the arrays are flattened. + +*Returns* : + A (p,) array of comparison values. + """ (med_1, med_2) = (mmedian(group_1, axis=axis), mmedian(group_2, axis=axis)) (std_1, std_2) = (stde_median(group_1, axis=axis), @@ -320,21 +325,22 @@ #.............................................................................. def rank_data(data, axis=None, use_missing=False): """Returns the rank (also known as order statistics) of each data point - along the given axis. - If some values are tied, their rank is averaged. - If some values are masked, their rank is set to 0 if use_missing is False, or - set to the average rank of the unmasked values if use_missing is True. +along the given axis. + +If some values are tied, their rank is averaged. +If some values are masked, their rank is set to 0 if use_missing is False, or +set to the average rank of the unmasked values if use_missing is True. -:Inputs: - data : sequence +*Parameters* : + data : {sequence} Input data. The data is transformed to a masked array - axis : integer *[None]* + axis : {integer} Axis along which to perform the ranking. If None, the array is first flattened. An exception is raised if the axis is specified for arrays with a dimension larger than 2 - use_missing : boolean *[False]* - Flag indicating whether the masked values have a rank of 0 (False) or - equal to the average rank of the unmasked values (True) + use_missing : {boolean} + Whether the masked values have a rank of 0 (False) or equal to the + average rank of the unmasked values (True). """ # def _rank1d(data, use_missing=False): Modified: trunk/scipy/sandbox/maskedarray/mrecords.py =================================================================== --- trunk/scipy/sandbox/maskedarray/mrecords.py 2007-10-03 00:06:40 UTC (rev 3389) +++ trunk/scipy/sandbox/maskedarray/mrecords.py 2007-10-03 01:46:24 UTC (rev 3390) @@ -38,8 +38,7 @@ reserved_fields = ['_data','_mask','_fieldmask', 'dtype'] def _getformats(data): - """Returns the formats of each array of arraylist as a comma-separated - string.""" + "Returns the formats of each array of arraylist as a comma-separated string." if hasattr(data,'dtype'): return ",".join([desc[1] for desc in data.dtype.descr]) @@ -56,9 +55,9 @@ return formats[:-1] def _checknames(descr, names=None): - """Checks that the field names of the descriptor `descr` are not some - reserved keywords. If this is the case, a default 'f%i' is substituted. - If the argument `names` is not None, updates the field names to valid names. + """Checks that the field names of the descriptor ``descr`` are not some +reserved keywords. If this is the case, a default 'f%i' is substituted. +If the argument `names` is not None, updates the field names to valid names. """ ndescr = len(descr) default_names = ['f%i' % i for i in range(ndescr)] @@ -90,12 +89,15 @@ class MaskedRecords(MaskedArray, object): """ -:IVariables: - - `__localfdict` : Dictionary - Dictionary of local fields (`f0_data`, `f0_mask`...) - - `__globalfdict` : Dictionary - Dictionary of global fields, as the combination of a `_data` and a `_mask`. - (`f0`) +*IVariables*: + _data : {recarray} + Underlying data, as a record array. + _mask : {boolean array} + Mask of the records. A record is masked when all its fields are masked. + _fieldmask : {boolean recarray} + Record array of booleans, setting the mask of each individual field of each record. + _fill_value : {record} + Filling values for each field. """ _defaultfieldmask = nomask _defaulthardmask = False @@ -183,6 +185,7 @@ #...................................................... def __getattribute__(self, attr): + "Returns the given attribute." try: # Returns a generic attribute return object.__getattribute__(self,attr) @@ -205,6 +208,7 @@ raise AttributeError,"No attribute '%s' !" % attr def __setattr__(self, attr, val): + "Sets the attribute attr to the value val." newattr = attr not in self.__dict__ try: # Is attr a generic attribute ? @@ -241,7 +245,7 @@ #............................................ def __getitem__(self, indx): """Returns all the fields sharing the same fieldname base. - The fieldname base is either `_data` or `_mask`.""" +The fieldname base is either `_data` or `_mask`.""" _localdict = self.__dict__ _data = self._data # We want a field ........ @@ -263,18 +267,12 @@ return obj #............................................ def __setitem__(self, indx, value): - """Sets the given record to value.""" + "Sets the given record to value." MaskedArray.__setitem__(self, indx, value) -# def __getslice__(self, i, j): -# """Returns the slice described by [i,j].""" -# _localdict = self.__dict__ -# return MaskedRecords(_localdict['_data'][i:j], -# mask=_localdict['_fieldmask'][i:j], -# dtype=self.dtype) -# + def __setslice__(self, i, j, value): - """Sets the slice described by [i,j] to `value`.""" + "Sets the slice described by [i,j] to `value`." _localdict = self.__dict__ d = self._data m = _localdict['_fieldmask'] @@ -305,6 +303,7 @@ #..................................................... def __setmask__(self, mask): + "Sets the mask." names = self.dtype.names fmask = self.__dict__['_fieldmask'] newmask = make_mask(mask, copy=False) @@ -328,10 +327,7 @@ #...................................................... def __str__(self): - """x.__str__() <==> str(x) -Calculates the string representation, using masked for fill if it is enabled. -Otherwise, fills with fill value. - """ + "Calculates the string representation." if self.size > 1: mstr = ["(%s)" % ",".join([str(i) for i in s]) for s in zip(*[getattr(self,f) for f in self.dtype.names])] @@ -342,10 +338,7 @@ return "(%s)" % ", ".join(mstr) def __repr__(self): - """x.__repr__() <==> repr(x) -Calculates the repr representation, using masked for fill if it is enabled. -Otherwise fill with fill value. - """ + "Calculates the repr representation." _names = self.dtype.names fmt = "%%%is : %%s" % (max([len(n) for n in _names])+4,) reprstr = [fmt % (f,getattr(self,f)) for f in self.dtype.names] @@ -367,11 +360,12 @@ return ndarray.view(self, obj) #...................................................... def filled(self, fill_value=None): - """Returns an array of the same class as `_data`, - with masked values filled with `fill_value`. + """Returns an array of the same class as ``_data``, with masked values +filled with ``fill_value``. If ``fill_value`` is None, ``self.fill_value`` is +used instead. + Subclassing is preserved. -If `fill_value` is None, uses self.fill_value. """ _localdict = self.__dict__ d = self._data @@ -419,28 +413,31 @@ names=None, titles=None, aligned=False, byteorder=None): """Creates a mrecarray from a (flat) list of masked arrays. -:Parameters: - - `arraylist` : Sequence - A list of (masked) arrays. Each element of the sequence is first converted - to a masked array if needed. If a 2D array is passed as argument, it is - processed line by line - - `dtype` : numeric.dtype - Data type descriptor. - - `shape` : Integer *[None]* - Number of records. If None, `shape` is defined from the shape of the first - array in the list. - - `formats` : +*Parameters*: + arraylist : {sequence} + A list of (masked) arrays. Each element of the sequence is first converted + to a masked array if needed. If a 2D array is passed as argument, it is + processed line by line + dtype : {numeric.dtype} + Data type descriptor. + {shape} : {integer} + Number of records. If None, ``shape`` is defined from the shape of the + first array in the list. + formats : {sequence} + Sequence of formats for each individual field. If None, the formats will + be autodetected by inspecting the fields and selecting the highest dtype + possible. + names : {sequence} + Sequence of the names of each field. + -titles : {sequence} (Description to write) - - `names` : - (description to write) - - `titles`: - (Description to write) - - `aligned`: Boolen *[False]* + aligned : {boolean} (Description to write, not used anyway) - - `byteorder`: Boolen *[None]* + byteorder: {boolean} (Description to write, not used anyway) - - + +*Notes*: + Lists of tuples should be preferred over lists of lists for faster processing. """ arraylist = [masked_array(x) for x in arraylist] # Define/check the shape..................... @@ -481,13 +478,31 @@ titles=None, aligned=False, byteorder=None): """Creates a MaskedRecords from a list of records. - The data in the same field can be heterogeneous, they will be promoted - to the highest data type. This method is intended for creating - smaller record arrays. If used to create large array without formats - defined, it can be slow. +*Parameters*: + arraylist : {sequence} + A list of (masked) arrays. Each element of the sequence is first converted + to a masked array if needed. If a 2D array is passed as argument, it is + processed line by line + dtype : {numeric.dtype} + Data type descriptor. + {shape} : {integer} + Number of records. If None, ``shape`` is defined from the shape of the + first array in the list. + formats : {sequence} + Sequence of formats for each individual field. If None, the formats will + be autodetected by inspecting the fields and selecting the highest dtype + possible. + names : {sequence} + Sequence of the names of each field. + -titles : {sequence} + (Description to write) + aligned : {boolean} + (Description to write, not used anyway) + byteorder: {boolean} + (Description to write, not used anyway) - If formats is None, then this will auto-detect formats. Use a list of - tuples rather than a list of lists for faster processing. +*Notes*: + Lists of tuples should be preferred over lists of lists for faster processing. """ # reclist is in fact a mrecarray ................. if isinstance(reclist, MaskedRecords): @@ -537,9 +552,9 @@ def _guessvartypes(arr): """Tries to guess the dtypes of the str_ ndarray `arr`, by testing element-wise - conversion. Returns a list of dtypes. - The array is first converted to ndarray. If the array is 2D, the test is - performed on the first line. An exception is raised if the file is 3D or more. +conversion. Returns a list of dtypes. +The array is first converted to ndarray. If the array is 2D, the test is performed +on the first line. An exception is raised if the file is 3D or more. """ vartypes = [] arr = numeric.asarray(arr) @@ -587,22 +602,22 @@ varnames=None, vartypes=None): """Creates a mrecarray from data stored in the file `filename`. -:Parameters: - - `filename` : file name/handle - Handle of an opened file. - - `delimitor` : Character *None* - Alphanumeric character used to separate columns in the file. - If None, any (group of) white spacestring(s) will be used. - - `commentchar` : String *['#']* - Alphanumeric character used to mark the start of a comment. - - `missingchar` : String *['']* - String indicating missing data, and used to create the masks. - - `varnames` : Sequence *[None]* - Sequence of the variable names. If None, a list will be created from - the first non empty line of the file. - - `vartypes` : Sequence *[None]* - Sequence of the variables dtypes. If None, the sequence will be estimated - from the first non-commented line. +*Parameters* : + filename : {file name/handle} + Handle of an opened file. + delimitor : {string} + Alphanumeric character used to separate columns in the file. + If None, any (group of) white spacestring(s) will be used. + commentchar : {string} + Alphanumeric character used to mark the start of a comment. + missingchar` : {string} + String indicating missing data, and used to create the masks. + varnames : {sequence} + Sequence of the variable names. If None, a list will be created from + the first non empty line of the file. + vartypes : {sequence} + Sequence of the variables dtypes. If None, it will be estimated from + the first non-commented line. Ultra simple: the varnames are in the header, one line""" Modified: trunk/scipy/sandbox/maskedarray/mstats.py =================================================================== --- trunk/scipy/sandbox/maskedarray/mstats.py 2007-10-03 00:06:40 UTC (rev 3389) +++ trunk/scipy/sandbox/maskedarray/mstats.py 2007-10-03 01:46:24 UTC (rev 3390) @@ -32,10 +32,17 @@ #####-------------------------------------------------------------------------- def winsorize(data, alpha=0.2): - """Returns a Winsorized version of the input array: the (alpha/2.) lowest - values are set to the (alpha/2.)th percentile, and the (alpha/2.) highest - values are set to the (1-alpha/2.)th percentile - Masked values are skipped. The input array is first flattened. + """Returns a Winsorized version of the input array. + +The (alpha/2.) lowest values are set to the (alpha/2.)th percentile, and +the (alpha/2.) highest values are set to the (1-alpha/2.)th percentile +Masked values are skipped. + +*Parameters*: + data : {ndarray} + Input data to Winsorize. The data is first flattened. + alpha : {float}, optional + Percentage of total Winsorization : alpha/2. on the left, alpha/2. on the right """ data = masked_array(data, copy=False).ravel() idxsort = data.argsort() @@ -47,16 +54,17 @@ #.............................................................................. def trim_both(data, proportiontocut=0.2, axis=None): """Trims the data by masking the int(trim*n) smallest and int(trim*n) largest - values of data along the given axis, where n is the number of unmasked values. +values of data along the given axis, where n is the number of unmasked values. -:Inputs: - data : MaskedArray +*Parameters*: + data : {ndarray} Data to trim. - trim : float *[0.2]* + proportiontocut : {float} Percentage of trimming. If n is the number of unmasked values before trimming, the number of values after trimming is (1-2*trim)*n. - axis : integer *[None]* - Axis along which to perform the trimming. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. """ #................... def _trim_1D(data, trim): @@ -80,16 +88,21 @@ #.............................................................................. def trim_tail(data, proportiontocut=0.2, tail='left', axis=None): """Trims the data by masking int(trim*n) values from ONE tail of the data - along the given axis, where n is the number of unmasked values. - -:Inputs: - data : MaskedArray +along the given axis, where n is the number of unmasked values. + +*Parameters*: + data : {ndarray} Data to trim. - trim : float *[0.2]* + proportiontocut : {float} Percentage of trimming. If n is the number of unmasked values before trimming, - the number of values after trimming is (1-2*trim)*n. - axis : integer *[None]* - Axis along which to perform the trimming. + the number of values after trimming is (1-trim)*n. + tail : {string} + Trimming direction, in ('left', 'right'). If left, the proportiontocut + lowest values are set to the corresponding percentile. If right, the + proportiontocut highest values are used instead. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. """ #................... def _trim_1D(data, trim, left): @@ -126,32 +139,34 @@ #.............................................................................. def trimmed_mean(data, proportiontocut=0.2, axis=None): """Returns the trimmed mean of the data along the given axis. Trimming is - performed on both ends of the distribution. +performed on both ends of the distribution. -:Inputs: - data : MaskedArray +*Parameters*: + data : {ndarray} Data to trim. - proportiontocut : float *[0.2]* + proportiontocut : {float} Proportion of the data to cut from each side of the data . As a result, (2*proportiontocut*n) values are actually trimmed. - axis : integer *[None]* - Axis along which to perform the trimming. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. """ return trim_both(data, proportiontocut=proportiontocut, axis=axis).mean(axis=axis) #.............................................................................. def trimmed_stde(data, proportiontocut=0.2, axis=None): """Returns the standard error of the trimmed mean for the input data, - along the given axis. Trimming is performed on both ends of the distribution. +along the given axis. Trimming is performed on both ends of the distribution. -:Inputs: - data : MaskedArray +*Parameters*: + data : {ndarray} Data to trim. - proportiontocut : float *[0.2]* + proportiontocut : {float} Proportion of the data to cut from each side of the data . As a result, (2*proportiontocut*n) values are actually trimmed. - axis : integer *[None]* - Axis along which to perform the trimming. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. """ #........................ def _trimmed_stde_1D(data, trim=0.2): @@ -172,7 +187,15 @@ #............................................................................. def stde_median(data, axis=None): """Returns the McKean-Schrader estimate of the standard error of the sample - median along the given axis. +median along the given axis. + + +*Parameters*: + data : {ndarray} + Data to trim. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. """ def _stdemed_1D(data): sorted = numpy.sort(data.compressed()) @@ -217,18 +240,18 @@ - (.4,.4) : approximately quantile unbiased (Cunnane) - (.35,.35): APL, used with PWM -:Parameters: - x : Sequence +*Parameters*: + x : {sequence} Input data, as a sequence or array of dimension at most 2. - prob : Sequence *[(0.25, 0.5, 0.75)]* + prob : {sequence} List of quantiles to compute. - alpha : Float (*[0.4]*) + alpha : {float} Plotting positions parameter. - beta : Float (*[0.4]*) + beta : {float} Plotting positions parameter. - axis : Integer *[None]* - Axis along which to compute quantiles. If *None*, uses the whole - (flattened/compressed) dataset. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. """ def _quantiles1D(data,m,p): x = numpy.sort(data.compressed()) @@ -302,23 +325,29 @@ def cov(x, y=None, rowvar=True, bias=False, strict=False): - """ - Estimate the covariance matrix. + """Estimates the covariance matrix. - If x is a vector, return the variance. For matrices, returns the covariance - matrix. - If y is given, it is treated as an additional (set of) variable(s). +Normalization is by (N-1) where N is the number of observations (unbiased +estimate). If bias is True then normalization is by N. - Normalization is by (N-1) where N is the number of observations (unbiased - estimate). If bias is True then normalization is by N. - - If rowvar is non-zero (default), then each row is a variable with observations - in the columns, otherwise each column is a variable and the observations are - in the rows. - - If strict is True, masked values are propagated: if a masked value appears in - a row or column, the whole row or column is considered masked. +*Parameters*: + x : {ndarray} + Input data. If x is a 1D array, returns the variance. If x is a 2D array, + returns the covariance matrix. + y : {ndarray}, optional + Optional set of variables. + rowvar : {boolean} + If rowvar is true, then each row is a variable with obersvations in columns. + If rowvar is False, each column is a variable and the observations are in + the rows. + bias : {boolean} + Whether to use a biased or unbiased estimate of the covariance. + If bias is True, then the normalization is by N, the number of observations. + Otherwise, the normalization is by (N-1) + strict : {boolean} + If strict is True, masked values are propagated: if a masked value appears in + a row or column, the whole row or column is considered masked. """ X = narray(x, ndmin=2, subok=True, dtype=float) if X.shape[0] == 1: @@ -350,7 +379,7 @@ def idealfourths(data, axis=None): """Returns an estimate of the interquartile range of the data along the given - axis, as computed with the ideal fourths. +axis, as computed with the ideal fourths. """ def _idf(data): x = numpy.sort(data.compressed()) @@ -368,13 +397,13 @@ def rsh(data, points=None): - """Evalutates Rosenblatt's shifted histogram estimators for each - point of 'points' on the dataset 'data'. + """Evalutates Rosenblatt's shifted histogram estimators for each point +on the dataset 'data'. -:Inputs: - data : sequence - Input data. Masked values are discarded. - points : +*Parameters* : + data : {sequence} + Input data. Masked values are ignored. + points : {sequence} Sequence of points where to evaluate Rosenblatt shifted histogram. If None, use the data. """ Modified: trunk/scipy/sandbox/maskedarray/testutils.py =================================================================== --- trunk/scipy/sandbox/maskedarray/testutils.py 2007-10-03 00:06:40 UTC (rev 3389) +++ trunk/scipy/sandbox/maskedarray/testutils.py 2007-10-03 01:46:24 UTC (rev 3390) @@ -22,13 +22,14 @@ from core import filled, equal, less #------------------------------------------------------------------------------ -def approx (a, b, fill_value=1, rtol=1.e-5, atol=1.e-8): +def approx (a, b, fill_value=True, rtol=1.e-5, atol=1.e-8): """Returns true if all components of a and b are equal subject to given tolerances. - If fill_value is 1, masked values considered equal. - If fill_value is 0, masked values considered unequal. - The relative error rtol should be positive and << 1.0 - The absolute error atol comes into play for those elements of b that are - very small or zero; it says how small a must be also. + +If fill_value is True, masked values considered equal. Otherwise, masked values +are considered unequal. +The relative error rtol should be positive and << 1.0 +The absolute error atol comes into play for those elements of b that are very +small or zero; it says how small a must be also. """ m = mask_or(getmask(a), getmask(b)) d1 = filled(a) @@ -183,8 +184,7 @@ header='Arrays are not equal') ##............................ def fail_if_array_equal(x, y, err_msg=''): - """Raises an assertion error if two masked arrays are not equal - (elem by elem.)""" + "Raises an assertion error if two masked arrays are not equal (elementwise)." def compare(x,y): return (not N.alltrue(approx(x, y))) From scipy-svn at scipy.org Wed Oct 3 01:58:36 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 00:58:36 -0500 (CDT) Subject: [Scipy-svn] r3391 - trunk/scipy/linalg/tests Message-ID: <20071003055836.CF45239C04C@new.scipy.org> Author: jarrod.millman Date: 2007-10-03 00:58:33 -0500 (Wed, 03 Oct 2007) New Revision: 3391 Modified: trunk/scipy/linalg/tests/test_decomp.py Log: fixing typo Modified: trunk/scipy/linalg/tests/test_decomp.py =================================================================== --- trunk/scipy/linalg/tests/test_decomp.py 2007-10-03 01:46:24 UTC (rev 3390) +++ trunk/scipy/linalg/tests/test_decomp.py 2007-10-03 05:58:33 UTC (rev 3391) @@ -500,7 +500,7 @@ """Check lu decomposition on medium size, rectangular matrix.""" self._test_common(self.cmed) -class test_lu_single(test_lu): +class TestLUSingle(TestLU): """LU testers for single precision, real and double""" def __init__(self, *args, **kw): test_lu.__init__(self, *args, **kw) @@ -519,7 +519,7 @@ self.med = self.vrect.astype(float32) self.cmed = self.vrect.astype(complex64) -class TestLuSolve(NumpyTestCase): +class TestLUSolve(NumpyTestCase): def check_lu(self): a = random((10,10)) b = random((10,)) @@ -531,7 +531,7 @@ assert_array_equal(x1,x2) -class TestSvd(NumpyTestCase): +class TestSVD(NumpyTestCase): def check_simple(self): a = [[1,2,3],[1,20,3],[2,5,6]] @@ -642,7 +642,7 @@ assert len(s)==2 assert s[0]>=s[1] -class TestDiagsvd(NumpyTestCase): +class TestDiagSVD(NumpyTestCase): def check_simple(self): assert_array_almost_equal(diagsvd([1,0,0],3,3),[[1,0,0],[0,0,0],[0,0,0]]) @@ -696,7 +696,7 @@ assert_array_almost_equal(cholesky(a,lower=1),c) -class TestQr(NumpyTestCase): +class TestQR(NumpyTestCase): def check_simple(self): a = [[8,2,3],[2,9,3],[5,3,6]] @@ -779,7 +779,7 @@ assert_array_almost_equal(dot(conj(transpose(q)),q),identity(n)) assert_array_almost_equal(dot(q,r),a) -class TestRq(NumpyTestCase): +class TestRQ(NumpyTestCase): def check_simple(self): a = [[8,2,3],[2,9,3],[5,3,6]] @@ -905,7 +905,7 @@ -class TestDatanotshared(NumpyTestCase): +class TestDataNotShared(NumpyTestCase): def check_datanotshared(self): from scipy.linalg.decomp import _datanotshared From scipy-svn at scipy.org Wed Oct 3 02:09:27 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 01:09:27 -0500 (CDT) Subject: [Scipy-svn] r3392 - trunk/scipy/linalg/tests Message-ID: <20071003060927.F18C139C166@new.scipy.org> Author: jarrod.millman Date: 2007-10-03 01:09:25 -0500 (Wed, 03 Oct 2007) New Revision: 3392 Modified: trunk/scipy/linalg/tests/test_decomp.py Log: typo Modified: trunk/scipy/linalg/tests/test_decomp.py =================================================================== --- trunk/scipy/linalg/tests/test_decomp.py 2007-10-03 05:58:33 UTC (rev 3391) +++ trunk/scipy/linalg/tests/test_decomp.py 2007-10-03 06:09:25 UTC (rev 3392) @@ -503,7 +503,7 @@ class TestLUSingle(TestLU): """LU testers for single precision, real and double""" def __init__(self, *args, **kw): - test_lu.__init__(self, *args, **kw) + TestLU.__init__(self, *args, **kw) self.a = self.a.astype(float32) self.ca = self.ca.astype(complex64) From scipy-svn at scipy.org Wed Oct 3 02:19:20 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 01:19:20 -0500 (CDT) Subject: [Scipy-svn] r3393 - in trunk/scipy/stats/models: . family tests Message-ID: <20071003061920.14AA439C08F@new.scipy.org> Author: jarrod.millman Date: 2007-10-03 01:19:08 -0500 (Wed, 03 Oct 2007) New Revision: 3393 Modified: trunk/scipy/stats/models/__init__.py trunk/scipy/stats/models/cox.py trunk/scipy/stats/models/family/links.py trunk/scipy/stats/models/formula.py trunk/scipy/stats/models/gam.py trunk/scipy/stats/models/glm.py trunk/scipy/stats/models/info.py trunk/scipy/stats/models/model.py trunk/scipy/stats/models/regression.py trunk/scipy/stats/models/rlm.py trunk/scipy/stats/models/smoothers.py trunk/scipy/stats/models/survival.py trunk/scipy/stats/models/tests/test_bspline.py trunk/scipy/stats/models/tests/test_formula.py trunk/scipy/stats/models/tests/test_glm.py trunk/scipy/stats/models/tests/test_regression.py Log: updating statistical models to use CapWords for class names Modified: trunk/scipy/stats/models/__init__.py =================================================================== --- trunk/scipy/stats/models/__init__.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/__init__.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -11,8 +11,8 @@ import scipy.stats.models.regression import scipy.stats.models.robust import scipy.stats.models.family -from scipy.stats.models.glm import model as glm -from scipy.stats.models.rlm import model as rlm +from scipy.stats.models.glm import Model as glm +from scipy.stats.models.rlm import Model as rlm __all__ = filter(lambda s:not s.startswith('_'),dir()) Modified: trunk/scipy/stats/models/cox.py =================================================================== --- trunk/scipy/stats/models/cox.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/cox.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -5,7 +5,7 @@ from scipy.stats.models import survival, model -class discrete: +class Discrete: """ A simple little class for working with discrete random vectors. @@ -37,7 +37,7 @@ dx = self.x - N.multiply.outer(mu, self.x.shape[1]) return N.dot(dx, N.transpose(dx)) -class observation(survival.right_censored): +class Observation(survival.RightCensored): def __getitem__(self, item): if self.namespace is not None: @@ -47,12 +47,13 @@ def __init__(self, time, delta, namespace=None): self.namespace = namespace - survival.right_censored.__init__(self, time, delta) + survival.RightCensored.__init__(self, time, delta) def __call__(self, formula, time=None, **extra): return formula(namespace=self, time=time, **extra) -class coxph(model.likelihood_model): +class CoxPH(model.LikelihoodModel): + """Cox proportional hazards regression model.""" def __init__(self, subjects, formula, time_dependent=False): self.subjects, self.formula = subjects, formula Modified: trunk/scipy/stats/models/family/links.py =================================================================== --- trunk/scipy/stats/models/family/links.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/family/links.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -134,7 +134,7 @@ g(x) = x**(1/self.power) INPUTS: - z -- linear predictors in glm + z -- linear predictors in GLM OUTPUTS: x x -- mean parameters @@ -227,7 +227,7 @@ g(x) = exp(x) INPUTS: - z -- linear predictors in glm + z -- linear predictors in GLM OUTPUTS: x x -- exp(z) @@ -289,7 +289,7 @@ g(z) = self.dbn.cdf(z) INPUTS: - z -- linear predictors in glm + z -- linear predictors in GLM OUTPUTS: p p -- inverse of CDF link of z Modified: trunk/scipy/stats/models/formula.py =================================================================== --- trunk/scipy/stats/models/formula.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/formula.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -9,7 +9,7 @@ default_namespace = {} -class term(object): +class Term(object): """ This class is very simple: it is just a named term in a model formula. @@ -35,7 +35,7 @@ else: name = '%s^%0.2f' % (self.name, power) - value = quantitative(name, func=self, transform=lambda x: N.power(x, power)) + value = Quantitative(name, func=self, transform=lambda x: N.power(x, power)) value.power = power value.namespace = self.namespace return value @@ -73,24 +73,24 @@ def __add__(self, other): """ - formula(self) + formula(other) + Formula(self) + Formula(other) """ - other = formula(other, namespace=self.namespace) + other = Formula(other, namespace=self.namespace) f = other + self f.namespace = self.namespace return f def __mul__(self, other): """ - formula(self) * formula(other) + Formula(self) * Formula(other) """ if other.name is 'intercept': - f = formula(self, namespace=self.namespace) + f = Formula(self, namespace=self.namespace) elif self.name is 'intercept': - f = formula(other, namespace=other.namespace) + f = Formula(other, namespace=other.namespace) else: - other = formula(other, namespace=self.namespace) + other = Formula(other, namespace=self.namespace) f = other * self f.namespace = self.namespace return f @@ -126,14 +126,9 @@ val = N.asarray(val) return N.squeeze(val) -class factor(term): +class Factor(Term): + """A categorical factor.""" - """ - A categorical factor. - """ - - - def __init__(self, termname, keys, ordinal=False): """ factor is initialized with keys, representing all valid @@ -151,7 +146,7 @@ else: name = ['(%s==%s)' % (self.termname, str(key)) for key in self.keys] - term.__init__(self, name, termname=self.termname, func=self.get_columns) + Term.__init__(self, name, termname=self.termname, func=self.get_columns) def get_columns(self, *args, **kw): """ @@ -199,18 +194,18 @@ def __add__(self, other): """ - formula(self) + formula(other) + Formula(self) + Formula(other) When adding \'intercept\' to a factor, this just returns - formula(self, namespace=self.namespace) + Formula(self, namespace=self.namespace) """ if other.name is 'intercept': - return formula(self, namespace=self.namespace) + return Formula(self, namespace=self.namespace) else: - return term.__add__(self, other) + return Term.__add__(self, other) def main_effect(self, reference=None): """ @@ -235,13 +230,13 @@ keep.pop(reference) __names = self.names() _names = ['%s-%s' % (__names[keep[i]], __names[reference]) for i in range(len(keep))] - value = quantitative(_names, func=self, + value = Quantitative(_names, func=self, termname='%s:maineffect' % self.termname, transform=maineffect_func) value.namespace = self.namespace return value -class quantitative(term): +class Quantitative(Term): """ A subclass of term that can be used to apply point transformations of another term, i.e. to take powers: @@ -249,29 +244,31 @@ >>> import numpy as N >>> from scipy.stats.models import formula >>> X = N.linspace(0,10,101) - >>> x = formula.term('X') + >>> x = formula.Term('X') >>> x.namespace={'X':X} >>> x2 = x**2 >>> print N.allclose(x()**2, x2()) True - >>> x3 = formula.quantitative('x2', func=x, transform=lambda x: x**2) + >>> x3 = formula.Quantitative('x2', func=x, transform=lambda x: x**2) >>> x3.namespace = x.namespace >>> print N.allclose(x()**2, x3()) True + """ def __init__(self, name, func=None, termname=None, transform=lambda x: x): self.transform = transform - term.__init__(self, name, func=func, termname=termname) + Term.__init__(self, name, func=func, termname=termname) def __call__(self, *args, **kw): """ A quantitative is just like term, except there is an additional transformation: self.transform. + """ - return self.transform(term.__call__(self, *args, **kw)) + return self.transform(Term.__call__(self, *args, **kw)) -class formula(object): +class Formula(object): """ A formula object for manipulating design matrices in regression models, essentially consisting of a list of term instances. @@ -279,6 +276,7 @@ The object supports addition and multiplication which correspond to concatenation and pairwise multiplication, respectively, of the columns of the two formulas. + """ def _get_namespace(self): @@ -304,11 +302,11 @@ self.__namespace = namespace - if isinstance(termlist, formula): + if isinstance(termlist, Formula): self.terms = copy.copy(list(termlist.terms)) elif type(termlist) is types.ListType: self.terms = termlist - elif isinstance(termlist, term): + elif isinstance(termlist, Term): self.terms = [termlist] else: raise ValueError @@ -388,11 +386,11 @@ Determine whether a given term is in a formula. """ - if not isinstance(query_term, formula): + if not isinstance(query_term, Formula): if type(query_term) == type("name"): try: query = self[query_term] except: return False - elif isinstance(query_term, term): + elif isinstance(query_term, Term): return query_term.termname in self.termnames() elif len(query_term.terms) == 1: query_term = query_term.terms[0] @@ -462,7 +460,7 @@ TO DO: check for nesting relationship. Should not be too difficult. """ - other = formula(other, namespace=self.namespace) + other = Formula(other, namespace=self.namespace) selftermnames = self.termnames() othertermnames = other.termnames() @@ -517,14 +515,14 @@ sumterms.terms = [self, other] # enforce the order we want sumterms.namespace = self.namespace - _term = quantitative(names, func=sumterms, termname=termname, + _term = Quantitative(names, func=sumterms, termname=termname, transform=product_func) _term.namespace = self.namespace terms.append(_term) - return formula(terms, namespace=self.namespace) + return Formula(terms, namespace=self.namespace) def __add__(self, other): @@ -535,12 +533,12 @@ terms in the formula are sorted alphabetically. """ - other = formula(other, namespace=self.namespace) + other = Formula(other, namespace=self.namespace) terms = self.terms + other.terms pieces = [(term.name, term) for term in terms] pieces.sort() terms = [piece[1] for piece in pieces] - return formula(terms, namespace=self.namespace) + return Formula(terms, namespace=self.namespace) def __sub__(self, other): @@ -550,7 +548,7 @@ function does not raise an exception. """ - other = formula(other, namespace=self.namespace) + other = Formula(other, namespace=self.namespace) terms = copy.copy(self.terms) for term in other.terms: @@ -558,7 +556,7 @@ if terms[i].termname == term.termname: terms.pop(i) break - return formula(terms, namespace=self.namespace) + return Formula(terms, namespace=self.namespace) def isnested(A, B, namespace=globals()): """ @@ -594,18 +592,19 @@ def _intercept_fn(nrow=1, **extra): return N.ones((1,nrow)) -I = term('intercept', func=_intercept_fn) + +I = Term('intercept', func=_intercept_fn) I.__doc__ = """ Intercept term in a formula. If intercept is the only term in the formula, then a keywords argument \'nrow\' is needed. ->>> from scipy.stats.models.formula import formula, I +>>> from scipy.stats.models.formula import Formula, I >>> I() array(1.0) >>> I(nrow=5) array([ 1., 1., 1., 1., 1.]) ->>> f=formula(I) +>>> f=Formula(I) >>> f(nrow=5) array([1, 1, 1, 1, 1]) Modified: trunk/scipy/stats/models/gam.py =================================================================== --- trunk/scipy/stats/models/gam.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/gam.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -7,7 +7,7 @@ from scipy.stats.models import family from scipy.stats.models.bspline import SmoothingSpline -from scipy.stats.models.glm import model as glm +from scipy.stats.models.glm import Model as GLM def default_smoother(x): _x = x.copy() @@ -37,7 +37,7 @@ s.target_df = 5 return s -class offset: +class Offset: def __init__(self, fn, offset): self.fn = fn @@ -46,7 +46,7 @@ def __call__(self, *args, **kw): return self.fn(*args, **kw) + self.offset -class results: +class Results: def __init__(self, Y, alpha, design, smoothers, family, offset): self.Y = Y @@ -67,7 +67,7 @@ def smoothed(self, design): return N.array([self.smoothers[i]() + self.offset[i] for i in range(design.shape[1])]) -class additive_model: +class AdditiveModel: def __init__(self, design, smoothers=None, weights=None): self.design = design @@ -141,13 +141,13 @@ return self.results -class model(glm, additive_model): +class Model(GLM, AdditiveModel): niter = 10 def __init__(self, design, smoothers=None, family=family.Gaussian()): - glm.__init__(self, design, family=family) - additive_model.__init__(self, design, smoothers=smoothers) + GLM.__init__(self, design, family=family) + AdditiveModel.__init__(self, design, smoothers=smoothers) self.family = family def next(self): @@ -155,7 +155,7 @@ _results.mu = self.family.link.inverse(_results.predict(self.design)) self.weights = self.family.weights(_results.mu) Z = _results.predict(self.design) + self.family.link.deriv(_results.mu) * (Y - _results.mu) - m = additive_model(self.design, smoothers=self.smoothers, weights=self.weights) + m = AdditiveModel(self.design, smoothers=self.smoothers, weights=self.weights) _results = m.fit(Z) _results.Y = Y _results.mu = self.family.link.inverse(_results.predict(self.design)) @@ -172,7 +172,7 @@ if Y is None: Y = self.Y resid = Y - self.results.mu - return (N.power(resid, 2) / self.family.variance(self.results.mu)).sum() / additive_model.df_resid(self) + return (N.power(resid, 2) / self.family.variance(self.results.mu)).sum() / AdditiveModel.df_resid(self) def fit(self, Y): self.Y = N.asarray(Y, N.float64) @@ -180,7 +180,7 @@ iter(self) alpha = self.Y.mean() Z = self.family.link(alpha) + self.family.link.deriv(alpha) * (Y - alpha) - m = additive_model(self.design, smoothers=self.smoothers) + m = AdditiveModel(self.design, smoothers=self.smoothers) self.results = m.fit(Z) self.results.mu = self.family.link.inverse(self.results.predict(self.design)) self.results.Y = Y @@ -209,7 +209,7 @@ y += z d = N.array([x1,x2]).T - m = additive_model(d) + m = AdditiveModel(d) m.fit(y) x = N.linspace(-2,2,50) Modified: trunk/scipy/stats/models/glm.py =================================================================== --- trunk/scipy/stats/models/glm.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/glm.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -1,18 +1,20 @@ """ General linear models -------------------- + """ + import numpy as N from scipy.stats.models import family -from scipy.stats.models.regression import wls_model +from scipy.stats.models.regression import WLSModel -class model(wls_model): +class Model(WLSModel): niter = 10 def __init__(self, design, family=family.Gaussian()): self.family = family - super(model, self).__init__(design, weights=1) + super(Model, self).__init__(design, weights=1) def __iter__(self): self.iter = 0 @@ -21,7 +23,7 @@ def deviance(self, Y=None, results=None, scale = 1.): """ - Return (unnormalized) log-likelihood for glm. + Return (unnormalized) log-likelihood for GLM. Note that self.scale is interpreted as a variance in old_model, so we divide the residuals by its sqrt. @@ -38,7 +40,7 @@ self.weights = self.family.weights(results.mu) self.initialize(self.design) Z = results.predict + self.family.link.deriv(results.mu) * (Y - results.mu) - newresults = super(model, self).fit(self, Z) + newresults = super(Model, self).fit(self, Z) newresults.Y = Y newresults.mu = self.family.link.inverse(newresults.predict) self.iter += 1 @@ -48,7 +50,7 @@ """ Continue iterating, or has convergence been obtained? """ - if self.iter >= model.niter: + if self.iter >= Model.niter: return False curdev = self.deviance(results=self.results) @@ -75,7 +77,7 @@ def fit(self, Y): self.Y = N.asarray(Y, N.float64) iter(self) - self.results = super(model, self).fit( + self.results = super(Model, self).fit( self.family.link.initialize(Y)) self.results.mu = self.family.link.inverse(self.results.predict) self.scale = self.results.scale = self.estimate_scale() Modified: trunk/scipy/stats/models/info.py =================================================================== --- trunk/scipy/stats/models/info.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/info.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -4,14 +4,14 @@ - model `formula` - standard `regression` models - - `ols_model` (ordinary least square regression) - - `wls_model` (weighted least square regression) - - `ar_model` (autoregressive model) + - `OLSModel` (ordinary least square regression) + - `WLSModel` (weighted least square regression) + - `ARModel` (autoregressive model) - - `glm.model` (generalized linear models) + - `glm.Model` (generalized linear models) - robust statistical models - - `rlm.model` (robust linear models using M estimators) + - `rlm.Model` (robust linear models using M estimators) - `robust.norms` estimates - `robust.scale` estimates (MAD, Huber's proposal 2). Modified: trunk/scipy/stats/models/model.py =================================================================== --- trunk/scipy/stats/models/model.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/model.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -41,7 +41,7 @@ """ raise NotImplementedError -class likelihood_model(Model): +class LikelihoodModel(Model): def logL(self, theta): """ @@ -69,7 +69,7 @@ # return -self.logL(theta) # self.results = optimize.fmin(f, theta) -class likelihood_model_results(object): +class LikelihoodModelResults(object): ''' Class to contain results from likelihood models ''' def __init__(self, beta, normalized_cov_beta=None, scale=1.): ''' Set up results structure Modified: trunk/scipy/stats/models/regression.py =================================================================== --- trunk/scipy/stats/models/regression.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/regression.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -22,11 +22,11 @@ import numpy.linalg as L from scipy.linalg import norm, toeplitz -from scipy.stats.models.model import likelihood_model, \ - likelihood_model_results +from scipy.stats.models.model import LikelihoodModel, \ + LikelihoodModelResults from scipy.stats.models import utils -class ols_model(likelihood_model): +class OLSModel(LikelihoodModel): """ A simple ordinary least squares model. @@ -34,15 +34,15 @@ -------- >>> import numpy as N >>> - >>> from scipy.stats.models.formula import term, I - >>> from scipy.stats.models.regression import ols_model + >>> from scipy.stats.models.formula import Term, I + >>> from scipy.stats.models.regression import OLSModel >>> >>> data={'Y':[1,3,4,5,2,3,4], ... 'X':range(1,8)} >>> f = term("X") + I >>> f.namespace = data >>> - >>> model = ols_model(f.design()) + >>> model = OLSModel(f.design()) >>> results = model.fit(data['Y']) >>> >>> results.beta @@ -60,13 +60,13 @@ def __init__(self, design): """ - Create a `ols_model` from a design. + Create a `OLSModel` from a design. :Parameters: design : TODO TODO """ - super(ols_model, self).__init__() + super(OLSModel, self).__init__() self.initialize(design) def initialize(self, design): @@ -100,7 +100,7 @@ """ Z = self.whiten(Y) - lfit = regression_results(L.lstsq(self.wdesign, Z)[0], Y) + lfit = RegressionResults(L.lstsq(self.wdesign, Z)[0], Y) lfit.predict = N.dot(self.design, lfit.beta) @@ -112,7 +112,7 @@ """ Z = self.whiten(Y) - lfit = regression_results(N.dot(self.calc_beta, Z), Y, + lfit = RegressionResults(N.dot(self.calc_beta, Z), Y, normalized_cov_beta=self.normalized_cov_beta) lfit.df_resid = self.df_resid @@ -124,7 +124,7 @@ return lfit -class ar_model(ols_model): +class ARModel(OLSModel): """ A regression model with an AR(p) covariance structure. @@ -136,20 +136,20 @@ >>> import numpy as N >>> import numpy.random as R >>> - >>> from scipy.stats.models.formula import term, I - >>> from scipy.stats.models.regression import ar_model + >>> from scipy.stats.models.formula import Term, I + >>> from scipy.stats.models.regression import ARModel >>> >>> data={'Y':[1,3,4,5,8,10,9], ... 'X':range(1,8)} >>> f = term("X") + I >>> f.namespace = data >>> - >>> model = ar_model(f.design(), 2) + >>> model = ARModel(f.design(), 2) >>> for i in range(6): ... results = model.fit(data['Y']) ... print "AR coefficients:", model.rho ... rho, sigma = model.yule_walker(data["Y"] - results.predict) - ... model = ar_model(model.design, rho) + ... model = ARModel(model.design, rho) ... AR coefficients: [ 0. 0.] AR coefficients: [-0.52571491 -0.84496178] @@ -182,7 +182,7 @@ if self.rho.shape == (): self.rho.shape = (1,) self.order = self.rho.shape[0] - super(ar_model, self).__init__(design) + super(ARModel, self).__init__(design) def iterative_fit(self, Y, niter=3): """ @@ -264,7 +264,7 @@ sigmasq = r[0] - (r[1:]*rho).sum() return rho, N.sqrt(sigmasq) -class wls_model(ols_model): +class WLSModel(OLSModel): """ A regression model with diagonal but non-identity covariance structure. The weights are presumed to be @@ -273,15 +273,15 @@ >>> import numpy as N >>> - >>> from scipy.stats.models.formula import term, I - >>> from scipy.stats.models.regression import wls_model + >>> from scipy.stats.models.formula import Term, I + >>> from scipy.stats.models.regression import WLSModel >>> >>> data={'Y':[1,3,4,5,2,3,4], ... 'X':range(1,8)} >>> f = term("X") + I >>> f.namespace = data >>> - >>> model = wls_model(f.design(), weights=range(1,8)) + >>> model = WLSModel(f.design(), weights=range(1,8)) >>> results = model.fit(data['Y']) >>> >>> results.beta @@ -304,7 +304,7 @@ raise ValueError( 'Weights must be scalar or same length as design') self.weights = weights.reshape(design_rows) - super(wls_model, self).__init__(design) + super(WLSModel, self).__init__(design) def whiten(self, X): """ @@ -321,7 +321,7 @@ v[:,i] = X[:,i] * c return v -class regression_results(likelihood_model_results): +class RegressionResults(LikelihoodModelResults): """ This class summarizes the fit of a linear regression model. @@ -329,7 +329,7 @@ """ def __init__(self, beta, Y, normalized_cov_beta=None, scale=1.): - super(regression_results, self).__init__(beta, + super(RegressionResults, self).__init__(beta, normalized_cov_beta, scale) self.Y = Y Modified: trunk/scipy/stats/models/rlm.py =================================================================== --- trunk/scipy/stats/models/rlm.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/rlm.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -3,10 +3,10 @@ """ import numpy as N -from scipy.stats.models.regression import wls_model +from scipy.stats.models.regression import WLSModel from scipy.stats.models.robust import norms, scale -class model(wls_model): +class Model(WLSModel): niter = 20 scale_est = 'MAD' @@ -25,7 +25,7 @@ """ Return (unnormalized) log-likelihood from M estimator. - Note that self.scale is interpreted as a variance in ols_model, so + Note that self.scale is interpreted as a variance in OLSModel, so we divide the residuals by its sqrt. """ if results is None: @@ -36,7 +36,7 @@ results = self.results self.weights = self.M.weights((results.Y - results.predict) / N.sqrt(results.scale)) self.initialize(self.design) - results = wls_model.fit(self, results.Y) + results = WLSModel.fit(self, results.Y) self.scale = results.scale = self.estimate_scale(results) self.iter += 1 return results @@ -45,7 +45,7 @@ """ Continue iterating, or has convergence been obtained? """ - if self.iter >= model.niter: + if self.iter >= Model.niter: return False curdev = self.deviance(results) @@ -57,7 +57,7 @@ def estimate_scale(self, results): """ - Note that self.scale is interpreted as a variance in ols_model, so + Note that self.scale is interpreted as a variance in OLSModel, so we return MAD(resid)**2 by default. """ resid = results.Y - results.predict @@ -71,7 +71,7 @@ def fit(self, Y): iter(self) - self.results = wls_model.fit(self, Y) + self.results = WLSModel.fit(self, Y) self.scale = self.results.scale = self.estimate_scale(self.results) while self.cont(self.results): Modified: trunk/scipy/stats/models/smoothers.py =================================================================== --- trunk/scipy/stats/models/smoothers.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/smoothers.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -13,7 +13,7 @@ from scipy.stats.models.bspline import bspline, _band2array -class poly_smoother: +class PolySmoother: """ Polynomial smoother up to a given order. Fit based on weighted least squares. @@ -52,7 +52,7 @@ _w = N.sqrt(weights) if x is None: if not hasattr(self, "X"): - raise ValueError, "x needed to fit poly_smoother" + raise ValueError, "x needed to fit PolySmoother" else: self.X = N.array([(x**i) for i in range(self.order+1)]) @@ -61,7 +61,7 @@ _y = y * _w self.coef = N.dot(L.pinv(X).T, _y) -class smoothing_spline(bspline): +class SmoothingSpline(bspline): penmax = 30. @@ -159,7 +159,7 @@ else: return self.rank -class smoothing_spline_fixeddf(smoothing_spline): +class SmoothingSplineFixedDF(SmoothingSpline): """ Fit smoothing spline with approximately df degrees of freedom used in the fit, i.e. so that self.trace() is approximately df. @@ -187,7 +187,7 @@ if not self.target_reached: while True: curpen = 0.5 * (apen + bpen) - smoothing_spline.fit(self, y, x=x, weights=weights, pen=curpen) + SmoothingSpline.fit(self, y, x=x, weights=weights, pen=curpen) curdf = self.trace() if curdf > df: apen, bpen = curpen, 2 * curpen @@ -199,9 +199,9 @@ self.target_reached = True break else: - smoothing_spline.fit(self, y, x=x, weights=weights, pen=self.pen) + SmoothingSpline.fit(self, y, x=x, weights=weights, pen=self.pen) -class smoothing_spline_gcv(smoothing_spline): +class SmoothingSplineGCV(SmoothingSpline): """ Fit smoothing spline trying to optimize GCV. @@ -218,7 +218,7 @@ bracket=(0,1.0e-03)): def _gcv(pen, y, x): - smoothing_spline.fit(y, x=x, pen=N.exp(pen), weights=weights) + SmoothingSpline.fit(y, x=x, pen=N.exp(pen), weights=weights) a = self.gcv() return a Modified: trunk/scipy/stats/models/survival.py =================================================================== --- trunk/scipy/stats/models/survival.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/survival.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -1,18 +1,18 @@ import numpy as N -class survival_time: +class SurvivalTime: def __init__(self, time, delta): self.time, self.delta = time, delta def atrisk(self, time): raise NotImplementedError -class right_censored(survival_time): +class RightCensored(SurvivalTime): def atrisk(self, time): return N.less_equal.outer(time, self.time) -class left_censored(survival_time): +class LeftCensored(SurvivalTime): def atrisk(self, time): return N.greater_equal.outer(time, self.time) Modified: trunk/scipy/stats/models/tests/test_bspline.py =================================================================== --- trunk/scipy/stats/models/tests/test_bspline.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/tests/test_bspline.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -1,5 +1,5 @@ """ -Test functions for models.glm +Test functions for models.bspline """ import numpy as N Modified: trunk/scipy/stats/models/tests/test_formula.py =================================================================== --- trunk/scipy/stats/models/tests/test_formula.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/tests/test_formula.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -14,38 +14,38 @@ class TestTerm(NumpyTestCase): def test_init(self): - t1 = formula.term("trivial") + t1 = formula.Term("trivial") sqr = lambda x: x*x - t2 = formula.term("not_so_trivial", sqr, "sqr") + t2 = formula.Term("not_so_trivial", sqr, "sqr") - self.assertRaises(ValueError, formula.term, "name", termname=0) + self.assertRaises(ValueError, formula.Term, "name", termname=0) def test_str(self): - t = formula.term("name") + t = formula.Term("name") s = str(t) def test_add(self): - t1 = formula.term("t1") - t2 = formula.term("t2") + t1 = formula.Term("t1") + t2 = formula.Term("t2") f = t1 + t2 - self.assert_(isinstance(f, formula.formula)) + self.assert_(isinstance(f, formula.Formula)) self.assert_(f.hasterm(t1)) self.assert_(f.hasterm(t2)) def test_mul(self): - t1 = formula.term("t1") - t2 = formula.term("t2") + t1 = formula.Term("t1") + t2 = formula.Term("t2") f = t1 * t2 - self.assert_(isinstance(f, formula.formula)) + self.assert_(isinstance(f, formula.Formula)) - intercept = formula.term("intercept") + intercept = formula.Term("intercept") f = t1 * intercept - self.assertEqual(str(f), str(formula.formula(t1))) + self.assertEqual(str(f), str(formula.Formula(t1))) f = intercept * t1 - self.assertEqual(str(f), str(formula.formula(t1))) + self.assertEqual(str(f), str(formula.Formula(t1))) class TestFormula(NumpyTestCase): @@ -56,7 +56,7 @@ for i in range(10): name = '%s' % string.uppercase[i] self.namespace[name] = self.X[:,i] - self.terms.append(formula.term(name)) + self.terms.append(formula.Term(name)) self.formula = self.terms[0] for i in range(1, 10): @@ -66,8 +66,8 @@ def test_namespace(self): space1 = {'X':N.arange(50), 'Y':N.arange(50)*2} space2 = {'X':N.arange(20), 'Y':N.arange(20)*2} - X = formula.term('X') - Y = formula.term('Y') + X = formula.Term('X') + Y = formula.Term('Y') X.namespace = space1 assert_almost_equal(X(), N.arange(50)) @@ -89,12 +89,12 @@ def test_termcolumns(self): - t1 = formula.term("A") - t2 = formula.term("B") + t1 = formula.Term("A") + t2 = formula.Term("B") f = t1 + t2 + t1 * t2 def other(val): return N.array([3.2*val,4.342*val**2, 5.234*val**3]) - q = formula.quantitative(['other%d' % i for i in range(1,4)], termname='other', func=t1, transform=other) + q = formula.Quantitative(['other%d' % i for i in range(1,4)], termname='other', func=t1, transform=other) f += q q.namespace = f.namespace = self.formula.namespace assert_almost_equal(q(), f()[f.termcolumns(q)]) @@ -150,7 +150,7 @@ def test_contrast2(self): - dummy = formula.term('zero') + dummy = formula.Term('zero') self.namespace['zero'] = N.zeros((40,), N.float64) term = dummy + self.terms[2] c = contrast.Contrast(term, self.formula) @@ -163,7 +163,7 @@ X = self.formula.design() P = N.dot(X, L.pinv(X)) - dummy = formula.term('noise') + dummy = formula.Term('noise') resid = N.identity(40) - P self.namespace['noise'] = N.transpose(N.dot(resid, R.standard_normal((40,5)))) terms = dummy + self.terms[2] @@ -181,32 +181,32 @@ def test_quantitative(self): t = self.terms[2] - sint = formula.quantitative('t', func=t, transform=N.sin) + sint = formula.Quantitative('t', func=t, transform=N.sin) t.namespace = sint.namespace = self.formula.namespace assert_almost_equal(N.sin(t()), sint()) def test_factor1(self): f = ['a','b','c']*10 - fac = formula.factor('ff', set(f)) + fac = formula.Factor('ff', set(f)) fac.namespace = {'ff':f} self.assertEquals(list(fac.values()), f) def test_factor2(self): f = ['a','b','c']*10 - fac = formula.factor('ff', set(f)) + fac = formula.Factor('ff', set(f)) fac.namespace = {'ff':f} self.assertEquals(fac().shape, (3,30)) def test_factor3(self): f = ['a','b','c']*10 - fac = formula.factor('ff', set(f)) + fac = formula.Factor('ff', set(f)) fac.namespace = {'ff':f} m = fac.main_effect(reference=1) self.assertEquals(m().shape, (2,30)) def test_factor4(self): f = ['a','b','c']*10 - fac = formula.factor('ff', set(f)) + fac = formula.Factor('ff', set(f)) fac.namespace = {'ff':f} m = fac.main_effect(reference=2) r = N.array([N.identity(3)]*10) Modified: trunk/scipy/stats/models/tests/test_glm.py =================================================================== --- trunk/scipy/stats/models/tests/test_glm.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/tests/test_glm.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -1,5 +1,5 @@ """ -Test functions for models.glm +Test functions for models.GLM """ import numpy as N Modified: trunk/scipy/stats/models/tests/test_regression.py =================================================================== --- trunk/scipy/stats/models/tests/test_regression.py 2007-10-03 06:09:25 UTC (rev 3392) +++ trunk/scipy/stats/models/tests/test_regression.py 2007-10-03 06:19:08 UTC (rev 3393) @@ -5,7 +5,7 @@ from numpy.random import standard_normal from numpy.testing import NumpyTest, NumpyTestCase -from scipy.stats.models.regression import ols_model, ar_model +from scipy.stats.models.regression import OLSModel, ARModel W = standard_normal @@ -14,14 +14,14 @@ def testOLS(self): X = W((40,10)) Y = W((40,)) - model = ols_model(design=X) + model = OLSModel(design=X) results = model.fit(Y) self.assertEquals(results.df_resid, 30) def testAR(self): X = W((40,10)) Y = W((40,)) - model = ar_model(design=X, rho=0.4) + model = ARModel(design=X, rho=0.4) results = model.fit(Y) self.assertEquals(results.df_resid, 30) @@ -29,7 +29,7 @@ X = W((40,10)) X[:,0] = X[:,1] + X[:,2] Y = W((40,)) - model = ols_model(design=X) + model = OLSModel(design=X) results = model.fit(Y) self.assertEquals(results.df_resid, 31) @@ -37,7 +37,7 @@ X = W((40,10)) X[:,0] = X[:,1] + X[:,2] Y = W((40,)) - model = ar_model(design=X, rho=0.9) + model = ARModel(design=X, rho=0.9) results = model.fit(Y) self.assertEquals(results.df_resid, 31) From scipy-svn at scipy.org Wed Oct 3 14:42:27 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 13:42:27 -0500 (CDT) Subject: [Scipy-svn] r3394 - trunk/scipy/sandbox/multigrid Message-ID: <20071003184227.CBC7039C1DB@new.scipy.org> Author: wnbell Date: 2007-10-03 13:42:23 -0500 (Wed, 03 Oct 2007) New Revision: 3394 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/coarsen.py trunk/scipy/sandbox/multigrid/multilevel.py Log: added support for multiple near-nullspace candidates in SA code Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-03 06:19:08 UTC (rev 3393) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-03 18:42:23 UTC (rev 3394) @@ -32,60 +32,8 @@ return Q,R -def fit_candidates(AggOp,candidates): - K = len(candidates) - N_fine,N_coarse = AggOp.shape - if len(candidates[0]) == K*N_fine: - #see if fine space has been expanded (all levels except for first) - AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) - N_fine = K*N_fine - - R = zeros((K*N_coarse,K)) - - candidate_matrices = [] - for i,c in enumerate(candidates): - X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) - - #TODO optimize this - - #orthogonalize X against previous - for j,A in enumerate(candidate_matrices): - D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X - R[j::K,i] = D_AtX - X.data -= D_AtX[X.indices] * A.data - - #AtX = csr_matrix(A.T.tocsr() * X - #R[j::K,i] = AtX.data - #X = X - A * AtX - - #normalize X - XtX = X.T.tocsr() * X - col_norms = sqrt(XtX.sum(axis=0)).flatten() - R[i::K,i] = col_norms - col_norms = 1.0/col_norms - col_norms[isinf(col_norms)] = 0 - X.data *= col_norms[X.indices] - - candidate_matrices.append(X) - - - Q_indptr = K*AggOp.indptr - Q_indices = (K*AggOp.indices).repeat(K) - for i in range(K): - Q_indices[i::K] += i - Q_data = empty(N_fine * K) - for i,X in enumerate(candidate_matrices): - Q_data[i::K] = X.data - Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) - - coarse_candidates = [R[:,i] for i in range(K)] - - return Q,coarse_candidates - - - ##def orthonormalize_candidate(I,x,basis): ## Px = csr_matrix((x,I.indices,I.indptr),dims=I.shape,check=False) ## Rs = [] Modified: trunk/scipy/sandbox/multigrid/coarsen.py =================================================================== --- trunk/scipy/sandbox/multigrid/coarsen.py 2007-10-03 06:19:08 UTC (rev 3393) +++ trunk/scipy/sandbox/multigrid/coarsen.py 2007-10-03 18:42:23 UTC (rev 3394) @@ -1,8 +1,11 @@ -import multigridtools -import scipy,numpy,scipy.sparse +import scipy +import numpy + +from numpy import arange,ones,zeros,sqrt,isinf,asarray,empty from scipy.sparse import csr_matrix,isspmatrix_csr from utils import diag_sparse,approximate_spectral_radius +import multigridtools def rs_strong_connections(A,theta): @@ -14,10 +17,9 @@ -A[i,j] >= theta * max( -A[i,k] ) where k != i """ if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') Sp,Sj,Sx = multigridtools.rs_strong_connections(A.shape[0],theta,A.indptr,A.indices,A.data) - return scipy.sparse.csr_matrix((Sx,Sj,Sp),A.shape) + return csr_matrix((Sx,Sj,Sp),A.shape) def rs_interpolation(A,theta=0.25): @@ -32,7 +34,7 @@ S.indptr,S.indices,S.data,\ T.indptr,T.indices,T.data) - return scipy.sparse.csr_matrix((Ix,Ij,Ip)) + return csr_matrix((Ix,Ij,Ip)) def sa_strong_connections(A,epsilon): @@ -40,11 +42,13 @@ Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) - return scipy.sparse.csr_matrix((Sx,Sj,Sp),A.shape) + return csr_matrix((Sx,Sj,Sp),A.shape) def sa_constant_interpolation(A,epsilon,blocks=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + #handle epsilon = 0 case without creating strength of connection matrix? + if blocks is not None: num_dofs = A.shape[0] num_blocks = blocks.max() @@ -71,9 +75,61 @@ Pp = numpy.arange(len(Pj)+1) Px = numpy.ones(len(Pj)) - return scipy.sparse.csr_matrix((Px,Pj,Pp)) + return csr_matrix((Px,Pj,Pp)) + +def fit_candidates(AggOp,candidates): + K = len(candidates) + + N_fine,N_coarse = AggOp.shape + + if K > 1 and len(candidates[0]) == K*N_fine: + #see if fine space has been expanded (all levels except for first) + AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) + N_fine = K*N_fine + + R = zeros((K*N_coarse,K)) + + candidate_matrices = [] + for i,c in enumerate(candidates): + X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) + + #TODO optimize this + + #orthogonalize X against previous + for j,A in enumerate(candidate_matrices): + D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X + R[j::K,i] = D_AtX + X.data -= D_AtX[X.indices] * A.data + + #AtX = csr_matrix(A.T.tocsr() * X + #R[j::K,i] = AtX.data + #X = X - A * AtX + #normalize X + XtX = X.T.tocsr() * X + col_norms = sqrt(asarray(XtX.sum(axis=0)).flatten()) + R[i::K,i] = col_norms + col_norms = 1.0/col_norms + col_norms[isinf(col_norms)] = 0 + X.data *= col_norms[X.indices] + + candidate_matrices.append(X) + + + Q_indptr = K*AggOp.indptr + Q_indices = (K*AggOp.indices).repeat(K) + for i in range(K): + Q_indices[i::K] += i + Q_data = empty(N_fine * K) + for i,X in enumerate(candidate_matrices): + Q_data[i::K] = X.data + Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) + + coarse_candidates = [R[:,i] for i in range(K)] + + return Q,coarse_candidates + ## S = sa_strong_connections(A,epsilon) ## ## #tentative (non-smooth) interpolation operator I @@ -88,19 +144,20 @@ ## ## return csr_matrix((Bx,Bj,Bp),dims=A.shape) -def sa_interpolation(A,epsilon,omega=4.0/3.0,blocks=None): +def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - P = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) + AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) + T,coarse_candidates = fit_candidates(AggOp,candidates) D_inv = diag_sparse(1.0/diag_sparse(A)) D_inv_A = D_inv * A D_inv_A *= omega/approximate_spectral_radius(D_inv_A) - I = P - (D_inv_A*P) #same as I=S*P, (faster?) + P = T - (D_inv_A*T) #same as I=S*P, (faster?) - return I + return P,coarse_candidates Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-03 06:19:08 UTC (rev 3393) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-03 18:42:23 UTC (rev 3394) @@ -58,7 +58,7 @@ return multilevel_solver(As,Ps) -def smoothed_aggregation_solver(A,blocks=None,max_levels=10,max_coarse=500,epsilon=0.08,omega=4.0/3.0): +def smoothed_aggregation_solver(A,candidates=None,blocks=None,max_levels=10,max_coarse=500,epsilon=0.08,omega=4.0/3.0): """ Create a multilevel solver using Smoothed Aggregation (SA) @@ -71,8 +71,11 @@ As = [A] Ps = [] + if candidates is None: + candidates = [ ones(A.shape[0]) ] # use constant vector + while len(As) < max_levels and A.shape[0] > max_coarse: - P = sa_interpolation(A,blocks=blocks,epsilon=epsilon*0.5**(len(As)-1),omega=omega) + P,candidates = sa_interpolation(A,candidates,epsilon*0.5**(len(As)-1),omega=omega,blocks=blocks) #P = sa_interpolation(A,epsilon=0.0) A = (P.T.tocsr() * A) * P #galerkin operator @@ -157,7 +160,7 @@ coarse_b = self.Ps[lvl].T * residual if lvl == len(self.As) - 2: - #direct solver on coarsest level + #use direct solver on coarsest level coarse_x[:] = scipy.linsolve.spsolve(self.As[-1],coarse_b) #coarse_x[:] = scipy.linalg.cg(self.As[-1],coarse_b,tol=1e-12)[0] #print "coarse residual norm",scipy.linalg.norm(coarse_b - self.As[-1]*coarse_x) @@ -170,29 +173,26 @@ def presmoother(self,A,x,b): - gauss_seidel(A,x,b,iterations=1,sweep="forward") - gauss_seidel(A,x,b,iterations=1,sweep="backward") - #sor(A,x,b,omega=1.85,iterations=1,sweep="backward") - - #x += 4.0/(3.0*infinity_norm(A))*(b - A*x) + gauss_seidel(A,x,b,iterations=1,sweep="symmetric") def postsmoother(self,A,x,b): - #sor(A,x,b,omega=1.85,iterations=1,sweep="forward") - gauss_seidel(A,x,b,iterations=1,sweep="forward") - gauss_seidel(A,x,b,iterations=1,sweep="backward") - #x += 4.0/(3.0*infinity_norm(A))*(b - A*x) + gauss_seidel(A,x,b,iterations=1,sweep="symmetric") if __name__ == '__main__': from scipy import * - #A = poisson_problem2D(100) + candidates = None + A = poisson_problem2D(100) #A = io.mmread("rocker_arm_surface.mtx").tocsr() #A = io.mmread("9pt-100x100.mtx").tocsr() - A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() + #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() - - ml = smoothed_aggregation_solver(A,max_coarse=100,max_levels=3) + #A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() + #candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') + #candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] + + ml = smoothed_aggregation_solver(A,candidates,max_coarse=100,max_levels=3) #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) From scipy-svn at scipy.org Wed Oct 3 17:38:00 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 16:38:00 -0500 (CDT) Subject: [Scipy-svn] r3395 - in trunk/scipy/sandbox/multigrid: . tests Message-ID: <20071003213800.5DFC739C0B1@new.scipy.org> Author: wnbell Date: 2007-10-03 16:37:54 -0500 (Wed, 03 Oct 2007) New Revision: 3395 Added: trunk/scipy/sandbox/multigrid/tests/test_sa.py Removed: trunk/scipy/sandbox/multigrid/tests/test_coarsen.py Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/coarsen.py trunk/scipy/sandbox/multigrid/multilevel.py Log: separated SA and RS codes Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-03 18:42:23 UTC (rev 3394) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-03 21:37:54 UTC (rev 3395) @@ -4,36 +4,36 @@ from relaxation import gauss_seidel from multilevel import multilevel_solver -from coarsen import sa_constant_interpolation +from sa import sa_constant_interpolation #from utils import infinity_norm from utils import approximate_spectral_radius -def fit_candidate(I,x): - """ - For each aggregate in I (i.e. each column of I) compute vector R and - sparse matrix Q (having the sparsity of I) such that the following holds: +##def fit_candidate(I,x): +## """ +## For each aggregate in I (i.e. each column of I) compute vector R and +## sparse matrix Q (having the sparsity of I) such that the following holds: +## +## Q*R = x and Q^T*Q = I +## +## In otherwords, find a prolongator Q with orthonormal columns so that +## x is represented exactly on the coarser level by R. +## """ +## x = asarray(x) +## Q = csr_matrix((x.copy(),I.indices,I.indptr),dims=I.shape,check=False) +## R = sqrt(ravel(csr_matrix((x*x,I.indices,I.indptr),dims=I.shape,check=False).sum(axis=0))) #column 2-norms +## +## Q.data *= (1.0/R)[Q.indices] #normalize columns of Q +## +## #print "norm(R)",scipy.linalg.norm(R) +## #print "min(R),max(R)",min(R),max(R) +## #print "infinity_norm(Q.T*Q - I) ",infinity_norm((Q.T.tocsr() * Q - scipy.sparse.spidentity(Q.shape[1]))) +## #print "norm(Q*R - x)",scipy.linalg.norm(Q*R - x) +## #print "norm(x - Q*Q.Tx)",scipy.linalg.norm(x - Q*(Q.T*x)) +## return Q,R - Q*R = x and Q^T*Q = I - In otherwords, find a prolongator Q with orthonormal columns so that - x is represented exactly on the coarser level by R. - """ - x = asarray(x) - Q = csr_matrix((x.copy(),I.indices,I.indptr),dims=I.shape,check=False) - R = sqrt(ravel(csr_matrix((x*x,I.indices,I.indptr),dims=I.shape,check=False).sum(axis=0))) #column 2-norms - Q.data *= (1.0/R)[Q.indices] #normalize columns of Q - - #print "norm(R)",scipy.linalg.norm(R) - #print "min(R),max(R)",min(R),max(R) - #print "infinity_norm(Q.T*Q - I) ",infinity_norm((Q.T.tocsr() * Q - scipy.sparse.spidentity(Q.shape[1]))) - #print "norm(Q*R - x)",scipy.linalg.norm(Q*R - x) - #print "norm(x - Q*Q.Tx)",scipy.linalg.norm(x - Q*(Q.T*x)) - return Q,R - - - ##def orthonormalize_candidate(I,x,basis): ## Px = csr_matrix((x,I.indices,I.indptr),dims=I.shape,check=False) ## Rs = [] @@ -62,7 +62,6 @@ V = concatenate((A.data,B.data)) return coo_matrix((V,(I,J)),dims=(A.shape[0],A.shape[1]+B.shape[1])).tocsr() - def vstack_csr(A,B): #OPTIMIZE THIS assert(A.shape[1] == B.shape[1]) @@ -145,7 +144,6 @@ Ps = [] for W in Ws: - #P,x = fit_candidate(W,x) P,x = fit_candidates(W,x) I = smoothed_prolongator(P,A) A = I.T.tocsr() * A * I Modified: trunk/scipy/sandbox/multigrid/coarsen.py =================================================================== --- trunk/scipy/sandbox/multigrid/coarsen.py 2007-10-03 18:42:23 UTC (rev 3394) +++ trunk/scipy/sandbox/multigrid/coarsen.py 2007-10-03 21:37:54 UTC (rev 3395) @@ -1,163 +1,163 @@ -import scipy -import numpy - -from numpy import arange,ones,zeros,sqrt,isinf,asarray,empty -from scipy.sparse import csr_matrix,isspmatrix_csr - -from utils import diag_sparse,approximate_spectral_radius -import multigridtools - - -def rs_strong_connections(A,theta): - """ - Return a strength of connection matrix using the method of Ruge and Stuben - - An off-diagonal entry A[i.j] is a strong connection iff - - -A[i,j] >= theta * max( -A[i,k] ) where k != i - """ - if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - - Sp,Sj,Sx = multigridtools.rs_strong_connections(A.shape[0],theta,A.indptr,A.indices,A.data) - return csr_matrix((Sx,Sj,Sp),A.shape) - - -def rs_interpolation(A,theta=0.25): - if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - - S = rs_strong_connections(A,theta) - - T = S.T.tocsr() #transpose S for efficient column access - - Ip,Ij,Ix = multigridtools.rs_interpolation(A.shape[0],\ - A.indptr,A.indices,A.data,\ - S.indptr,S.indices,S.data,\ - T.indptr,T.indices,T.data) - - return csr_matrix((Ix,Ij,Ip)) - - -def sa_strong_connections(A,epsilon): - if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - - Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) - - return csr_matrix((Sx,Sj,Sp),A.shape) - -def sa_constant_interpolation(A,epsilon,blocks=None): - if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - - #handle epsilon = 0 case without creating strength of connection matrix? - - if blocks is not None: - num_dofs = A.shape[0] - num_blocks = blocks.max() - - if num_dofs != len(blocks): - raise ValueError,'improper block specification' - - # for non-scalar problems, use pre-defined blocks in aggregation - # the strength of connection matrix is based on the Frobenius norms of the blocks - - B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) - Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B #Frobenius norms of blocks entries of A - - S = sa_strong_connections(Block_Frob,epsilon) - - Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) - Pj = Pj[blocks] #expand block aggregates into constituent dofs - Pp = B.indptr - Px = B.data - else: - S = sa_strong_connections(A,epsilon) - - Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) - Pp = numpy.arange(len(Pj)+1) - Px = numpy.ones(len(Pj)) - - return csr_matrix((Px,Pj,Pp)) - - -def fit_candidates(AggOp,candidates): - K = len(candidates) - - N_fine,N_coarse = AggOp.shape - - if K > 1 and len(candidates[0]) == K*N_fine: - #see if fine space has been expanded (all levels except for first) - AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) - N_fine = K*N_fine - - R = zeros((K*N_coarse,K)) - - candidate_matrices = [] - for i,c in enumerate(candidates): - X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) - - #TODO optimize this - - #orthogonalize X against previous - for j,A in enumerate(candidate_matrices): - D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X - R[j::K,i] = D_AtX - X.data -= D_AtX[X.indices] * A.data - - #AtX = csr_matrix(A.T.tocsr() * X - #R[j::K,i] = AtX.data - #X = X - A * AtX - - #normalize X - XtX = X.T.tocsr() * X - col_norms = sqrt(asarray(XtX.sum(axis=0)).flatten()) - R[i::K,i] = col_norms - col_norms = 1.0/col_norms - col_norms[isinf(col_norms)] = 0 - X.data *= col_norms[X.indices] - - candidate_matrices.append(X) - - - Q_indptr = K*AggOp.indptr - Q_indices = (K*AggOp.indices).repeat(K) - for i in range(K): - Q_indices[i::K] += i - Q_data = empty(N_fine * K) - for i,X in enumerate(candidate_matrices): - Q_data[i::K] = X.data - Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) - - coarse_candidates = [R[:,i] for i in range(K)] - - return Q,coarse_candidates - -## S = sa_strong_connections(A,epsilon) +##import scipy +##import numpy ## -## #tentative (non-smooth) interpolation operator I -## Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) -## Pp = numpy.arange(len(Pj)+1) -## Px = numpy.ones(len(Pj)) +##from numpy import arange,ones,zeros,sqrt,isinf,asarray,empty +##from scipy.sparse import csr_matrix,isspmatrix_csr +## +##from utils import diag_sparse,approximate_spectral_radius +##import multigridtools +## +## +##def rs_strong_connections(A,theta): +## """ +## Return a strength of connection matrix using the method of Ruge and Stuben +## +## An off-diagonal entry A[i.j] is a strong connection iff +## +## -A[i,j] >= theta * max( -A[i,k] ) where k != i +## """ +## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') +## +## Sp,Sj,Sx = multigridtools.rs_strong_connections(A.shape[0],theta,A.indptr,A.indices,A.data) +## return csr_matrix((Sx,Sj,Sp),A.shape) +## +## +##def rs_interpolation(A,theta=0.25): +## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') ## -## return scipy.sparse.csr_matrix((Px,Pj,Pp)) - -##def sa_smoother(A,S,omega): -## Bp,Bj,Bx = multigridtools.sa_smoother(A.shape[0],omega,A.indptr,A.indices,A.data,S.indptr,S.indices,S.data) +## S = rs_strong_connections(A,theta) ## -## return csr_matrix((Bx,Bj,Bp),dims=A.shape) - -def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): - if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - - AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) - T,coarse_candidates = fit_candidates(AggOp,candidates) - - D_inv = diag_sparse(1.0/diag_sparse(A)) - - D_inv_A = D_inv * A - D_inv_A *= omega/approximate_spectral_radius(D_inv_A) - - P = T - (D_inv_A*T) #same as I=S*P, (faster?) - - return P,coarse_candidates - - - +## T = S.T.tocsr() #transpose S for efficient column access +## +## Ip,Ij,Ix = multigridtools.rs_interpolation(A.shape[0],\ +## A.indptr,A.indices,A.data,\ +## S.indptr,S.indices,S.data,\ +## T.indptr,T.indices,T.data) +## +## return csr_matrix((Ix,Ij,Ip)) +## +## +##def sa_strong_connections(A,epsilon): +## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') +## +## Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) +## +## return csr_matrix((Sx,Sj,Sp),A.shape) +## +##def sa_constant_interpolation(A,epsilon,blocks=None): +## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') +## +## #handle epsilon = 0 case without creating strength of connection matrix? +## +## if blocks is not None: +## num_dofs = A.shape[0] +## num_blocks = blocks.max() +## +## if num_dofs != len(blocks): +## raise ValueError,'improper block specification' +## +## # for non-scalar problems, use pre-defined blocks in aggregation +## # the strength of connection matrix is based on the Frobenius norms of the blocks +## +## B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) +## Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B #Frobenius norms of blocks entries of A +## +## S = sa_strong_connections(Block_Frob,epsilon) +## +## Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) +## Pj = Pj[blocks] #expand block aggregates into constituent dofs +## Pp = B.indptr +## Px = B.data +## else: +## S = sa_strong_connections(A,epsilon) +## +## Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) +## Pp = numpy.arange(len(Pj)+1) +## Px = numpy.ones(len(Pj)) +## +## return csr_matrix((Px,Pj,Pp)) +## +## +##def fit_candidates(AggOp,candidates): +## K = len(candidates) +## +## N_fine,N_coarse = AggOp.shape +## +## if K > 1 and len(candidates[0]) == K*N_fine: +## #see if fine space has been expanded (all levels except for first) +## AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) +## N_fine = K*N_fine +## +## R = zeros((K*N_coarse,K)) +## +## candidate_matrices = [] +## for i,c in enumerate(candidates): +## X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) +## +## #TODO optimize this +## +## #orthogonalize X against previous +## for j,A in enumerate(candidate_matrices): +## D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X +## R[j::K,i] = D_AtX +## X.data -= D_AtX[X.indices] * A.data +## +## #AtX = csr_matrix(A.T.tocsr() * X +## #R[j::K,i] = AtX.data +## #X = X - A * AtX +## +## #normalize X +## XtX = X.T.tocsr() * X +## col_norms = sqrt(asarray(XtX.sum(axis=0)).flatten()) +## R[i::K,i] = col_norms +## col_norms = 1.0/col_norms +## col_norms[isinf(col_norms)] = 0 +## X.data *= col_norms[X.indices] +## +## candidate_matrices.append(X) +## +## +## Q_indptr = K*AggOp.indptr +## Q_indices = (K*AggOp.indices).repeat(K) +## for i in range(K): +## Q_indices[i::K] += i +## Q_data = empty(N_fine * K) +## for i,X in enumerate(candidate_matrices): +## Q_data[i::K] = X.data +## Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) +## +## coarse_candidates = [R[:,i] for i in range(K)] +## +## return Q,coarse_candidates +## +#### S = sa_strong_connections(A,epsilon) +#### +#### #tentative (non-smooth) interpolation operator I +#### Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) +#### Pp = numpy.arange(len(Pj)+1) +#### Px = numpy.ones(len(Pj)) +#### +#### return scipy.sparse.csr_matrix((Px,Pj,Pp)) +## +####def sa_smoother(A,S,omega): +#### Bp,Bj,Bx = multigridtools.sa_smoother(A.shape[0],omega,A.indptr,A.indices,A.data,S.indptr,S.indices,S.data) +#### +#### return csr_matrix((Bx,Bj,Bp),dims=A.shape) +## +##def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): +## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') +## +## AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) +## T,coarse_candidates = fit_candidates(AggOp,candidates) +## +## D_inv = diag_sparse(1.0/diag_sparse(A)) +## +## D_inv_A = D_inv * A +## D_inv_A *= omega/approximate_spectral_radius(D_inv_A) +## +## P = T - (D_inv_A*T) #same as I=S*P, (faster?) +## +## return P,coarse_candidates +## +## +## Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-03 18:42:23 UTC (rev 3394) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-03 21:37:54 UTC (rev 3395) @@ -2,12 +2,13 @@ 'ruge_stuben_solver','smoothed_aggregation_solver', 'multilevel_solver'] -from numpy.linalg import norm -from numpy import zeros,zeros_like,array import scipy import numpy +from numpy import zeros,zeros_like,array +from numpy.linalg import norm -from coarsen import sa_interpolation,rs_interpolation +from sa import sa_interpolation +from rs import rs_interpolation from relaxation import gauss_seidel,jacobi,sor from utils import infinity_norm @@ -188,11 +189,12 @@ #A = io.mmread("9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() + #A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() #candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') #candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] - ml = smoothed_aggregation_solver(A,candidates,max_coarse=100,max_levels=3) + ml = smoothed_aggregation_solver(A,candidates,max_coarse=10,max_levels=10) #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) Deleted: trunk/scipy/sandbox/multigrid/tests/test_coarsen.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_coarsen.py 2007-10-03 18:42:23 UTC (rev 3394) +++ trunk/scipy/sandbox/multigrid/tests/test_coarsen.py 2007-10-03 21:37:54 UTC (rev 3395) @@ -1,161 +0,0 @@ -from numpy.testing import * - -from numpy import sqrt,empty,ones,arange,array_split -from scipy import rand -from scipy.sparse import spdiags,csr_matrix,lil_matrix -import numpy - -set_package_path() -import scipy.sandbox.multigrid -from scipy.sandbox.multigrid.coarsen import sa_strong_connections,sa_constant_interpolation -from scipy.sandbox.multigrid.multilevel import poisson_problem1D,poisson_problem2D -restore_path() - - -def reference_sa_strong_connections(A,epsilon): - A_coo = A.tocoo() - S = lil_matrix(A.shape) - - for (i,j,v) in zip(A_coo.row,A_coo.col,A_coo.data): - if i == j: continue #skip diagonal - - if abs(A[i,j]) >= epsilon*sqrt(abs(A[i,i])*abs(A[j,j])): - S[i,j] = v - - return S.tocsr() - - -# note that this method only tests the current implementation, not -# all possible implementations -def reference_sa_constant_interpolation(A,epsilon): - S = sa_strong_connections(A,epsilon) - S = array_split(S.indices,S.indptr[1:-1]) - - n = A.shape[0] - - R = set(range(n)) - j = 0 - - aggregates = empty(n,dtype=A.indices.dtype) - aggregates[:] = -1 - - # Pass #1 - for i,row in enumerate(S): - Ni = set(row) | set([i]) - - if Ni.issubset(R): - R -= Ni - for x in Ni: - aggregates[x] = j - j += 1 - - # Pass #2 - Old_R = R.copy() - for i,row in enumerate(S): - if i not in R: continue - - for x in row: - if x not in Old_R: - aggregates[i] = aggregates[x] - R.remove(i) - break - - - # Pass #3 - for i,row in enumerate(S): - if i not in R: continue - Ni = set(row) | set([i]) - - for x in Ni: - if x in R: - aggregates[x] = j - j += 1 - - assert(len(R) == 0) - - Pj = aggregates - Pp = arange(n+1) - Px = ones(n) - - return csr_matrix((Px,Pj,Pp)) - -class TestSaStrongConnections(NumpyTestCase): - def check_simple(self): - N = 4 - A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() - S = spdiags([ -ones(N),-ones(N)],[-1,1],N,N).tocsr() - assert_array_equal(sa_strong_connections(A,0.50).todense(),S.todense()) #all connections are strong - assert_array_equal(sa_strong_connections(A,0.51).todense(),0*S.todense()) #no connections are strong - - N = 100 - A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() - S = spdiags([ -ones(N),-ones(N)],[-1,1],N,N).tocsr() - assert_array_equal(sa_strong_connections(A,0.50).todense(),S.todense()) #all connections are strong - assert_array_equal(sa_strong_connections(A,0.51).todense(),0*S.todense()) #no connections are strong - - def check_random(self): - numpy.random.seed(0) - - for N in [2,3,5,10]: - A = csr_matrix(rand(N,N)) - for epsilon in [0.0,0.1,0.5,0.8,1.0,10.0]: - S_result = sa_strong_connections(A,epsilon) - S_expected = reference_sa_strong_connections(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - - def check_poisson1D(self): - for N in [2,3,5,7,10,11,19]: - A = poisson_problem1D(N) - for epsilon in [0.0,0.1,0.5,0.8,1.0]: - S_result = sa_strong_connections(A,epsilon) - S_expected = reference_sa_strong_connections(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - - def check_poisson2D(self): - for N in [2,3,5,7,10,11,19]: - A = poisson_problem2D(N) - for epsilon in [0.0,0.1,0.5,0.8,1.0]: - S_result = sa_strong_connections(A,epsilon) - S_expected = reference_sa_strong_connections(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - - -class TestSaConstantInterpolation(NumpyTestCase): - def check_random(self): - numpy.random.seed(0) - for N in [2,3,5,10]: - A = csr_matrix(rand(N,N)) - for epsilon in [0.0,0.1,0.5,0.8,1.0]: - S_result = sa_constant_interpolation(A,epsilon) - S_expected = reference_sa_constant_interpolation(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - - def check_poisson1D(self): - for N in [2,3,5,7,10,11,20,21,29,30]: - A = poisson_problem1D(N) - for epsilon in [0.0,0.1,0.5,0.8,1.0]: - S_result = sa_constant_interpolation(A,epsilon) - S_expected = reference_sa_constant_interpolation(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - - def check_poisson2D(self): - for N in [2,3,5,7,10,11,20,21,29,30]: - A = poisson_problem2D(N) - for epsilon in [0.0,0.1,0.5,0.8,1.0]: - S_result = sa_constant_interpolation(A,epsilon) - S_expected = reference_sa_constant_interpolation(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - -## def check_sample_data(self): -## from examples import all_examples,read_matrix -## -## for filename in all_examples: -## A = read_matrix(filename) -## for epsilon in [0.0,0.08,0.51,1.0]: -## S_result = sa_constant_interpolation(A,epsilon) -## S_expected = reference_sa_constant_interpolation(A,epsilon) -## assert_array_equal((S_result - S_expected).nnz,0) - -if __name__ == '__main__': - NumpyTest().run() - Copied: trunk/scipy/sandbox/multigrid/tests/test_sa.py (from rev 3393, trunk/scipy/sandbox/multigrid/tests/test_coarsen.py) =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_coarsen.py 2007-10-03 06:19:08 UTC (rev 3393) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-03 21:37:54 UTC (rev 3395) @@ -0,0 +1,160 @@ +from numpy.testing import * + +from numpy import sqrt,empty,ones,arange,array_split +from scipy import rand +from scipy.sparse import spdiags,csr_matrix,lil_matrix +import numpy + +set_package_path() +import scipy.sandbox.multigrid +from scipy.sandbox.multigrid.sa import sa_strong_connections, sa_constant_interpolation, \ + sa_interpolation, sa_fit_candidates +from scipy.sandbox.multigrid.multilevel import poisson_problem1D,poisson_problem2D +restore_path() + + +def reference_sa_strong_connections(A,epsilon): + A_coo = A.tocoo() + S = lil_matrix(A.shape) + + for (i,j,v) in zip(A_coo.row,A_coo.col,A_coo.data): + if i == j: continue #skip diagonal + + if abs(A[i,j]) >= epsilon*sqrt(abs(A[i,i])*abs(A[j,j])): + S[i,j] = v + + return S.tocsr() + +class TestSAStrongConnections(NumpyTestCase): + def check_simple(self): + N = 4 + A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() + S = spdiags([ -ones(N),-ones(N)],[-1,1],N,N).tocsr() + assert_array_equal(sa_strong_connections(A,0.50).todense(),S.todense()) #all connections are strong + assert_array_equal(sa_strong_connections(A,0.51).todense(),0*S.todense()) #no connections are strong + + N = 100 + A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() + S = spdiags([ -ones(N),-ones(N)],[-1,1],N,N).tocsr() + assert_array_equal(sa_strong_connections(A,0.50).todense(),S.todense()) #all connections are strong + assert_array_equal(sa_strong_connections(A,0.51).todense(),0*S.todense()) #no connections are strong + + def check_random(self): + numpy.random.seed(0) + + for N in [2,3,5]: + A = csr_matrix(rand(N,N)) + for epsilon in [0.0,0.1,0.5,1.0,10.0]: + S_result = sa_strong_connections(A,epsilon) + S_expected = reference_sa_strong_connections(A,epsilon) + assert_array_equal(S_result.todense(),S_expected.todense()) + + def check_poisson1D(self): + for N in [2,3,5,7,10,11,19]: + A = poisson_problem1D(N) + for epsilon in [0.0,0.1,0.5,1.0]: + S_result = sa_strong_connections(A,epsilon) + S_expected = reference_sa_strong_connections(A,epsilon) + assert_array_equal(S_result.todense(),S_expected.todense()) + + def check_poisson2D(self): + for N in [2,3,5,7,10,11]: + A = poisson_problem2D(N) + for epsilon in [0.0,0.1,0.5,1.0]: + S_result = sa_strong_connections(A,epsilon) + S_expected = reference_sa_strong_connections(A,epsilon) + assert_array_equal(S_result.todense(),S_expected.todense()) + + +# note that this method only tests the current implementation, not +# all possible implementations +def reference_sa_constant_interpolation(A,epsilon): + S = sa_strong_connections(A,epsilon) + S = array_split(S.indices,S.indptr[1:-1]) + + n = A.shape[0] + + R = set(range(n)) + j = 0 + + aggregates = empty(n,dtype=A.indices.dtype) + aggregates[:] = -1 + + # Pass #1 + for i,row in enumerate(S): + Ni = set(row) | set([i]) + + if Ni.issubset(R): + R -= Ni + for x in Ni: + aggregates[x] = j + j += 1 + + # Pass #2 + Old_R = R.copy() + for i,row in enumerate(S): + if i not in R: continue + + for x in row: + if x not in Old_R: + aggregates[i] = aggregates[x] + R.remove(i) + break + + # Pass #3 + for i,row in enumerate(S): + if i not in R: continue + Ni = set(row) | set([i]) + + for x in Ni: + if x in R: + aggregates[x] = j + j += 1 + + assert(len(R) == 0) + + Pj = aggregates + Pp = arange(n+1) + Px = ones(n) + + return csr_matrix((Px,Pj,Pp)) + +class TestSAConstantInterpolation(NumpyTestCase): + def check_random(self): + numpy.random.seed(0) + for N in [2,3,5,10]: + A = csr_matrix(rand(N,N)) + for epsilon in [0.0,0.1,0.5,1.0]: + S_result = sa_constant_interpolation(A,epsilon) + S_expected = reference_sa_constant_interpolation(A,epsilon) + assert_array_equal(S_result.todense(),S_expected.todense()) + + def check_poisson1D(self): + for N in [2,3,5,7,10,11,20,21,29,30]: + A = poisson_problem1D(N) + for epsilon in [0.0,0.1,0.5,1.0]: + S_result = sa_constant_interpolation(A,epsilon) + S_expected = reference_sa_constant_interpolation(A,epsilon) + assert_array_equal(S_result.todense(),S_expected.todense()) + + def check_poisson2D(self): + for N in [2,3,5,7,10,11]: + A = poisson_problem2D(N) + for epsilon in [0.0,0.1,0.5,1.0]: + S_result = sa_constant_interpolation(A,epsilon) + S_expected = reference_sa_constant_interpolation(A,epsilon) + assert_array_equal(S_result.todense(),S_expected.todense()) + +## def check_sample_data(self): +## from examples import all_examples,read_matrix +## +## for filename in all_examples: +## A = read_matrix(filename) +## for epsilon in [0.0,0.08,0.51,1.0]: +## S_result = sa_constant_interpolation(A,epsilon) +## S_expected = reference_sa_constant_interpolation(A,epsilon) +## assert_array_equal((S_result - S_expected).nnz,0) + +if __name__ == '__main__': + NumpyTest().run() + From scipy-svn at scipy.org Wed Oct 3 18:43:46 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 17:43:46 -0500 (CDT) Subject: [Scipy-svn] r3396 - trunk/scipy/sandbox/multigrid/tests Message-ID: <20071003224346.5FFD139C075@new.scipy.org> Author: wnbell Date: 2007-10-03 17:43:43 -0500 (Wed, 03 Oct 2007) New Revision: 3396 Modified: trunk/scipy/sandbox/multigrid/tests/test_adaptive.py trunk/scipy/sandbox/multigrid/tests/test_sa.py Log: added unittests for sa_fit_candidates Modified: trunk/scipy/sandbox/multigrid/tests/test_adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_adaptive.py 2007-10-03 21:37:54 UTC (rev 3395) +++ trunk/scipy/sandbox/multigrid/tests/test_adaptive.py 2007-10-03 22:43:43 UTC (rev 3396) @@ -4,49 +4,14 @@ from scipy import arange,ones,zeros,array,eye set_package_path() -from scipy.sandbox.multigrid.adaptive import fit_candidates +pass restore_path() -class TestFitCandidates(NumpyTestCase): +class TestAdaptiveSA(NumpyTestCase): def setUp(self): - self.cases = [] - - #one candidate - self.cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)),[ones(5)])) - self.cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)),[ones(5)])) - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9)])) - self.cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)),[arange(9)])) - - #two candidates - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)),[ones(4),arange(4)])) - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9),arange(9)])) - self.cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)),[ones(9),arange(9)])) - - def check_all(self): - for AggOp,fine_candidates in self.cases: - Q,coarse_candidates = fit_candidates(AggOp,fine_candidates) - - assert_equal(len(coarse_candidates),len(fine_candidates)) - assert_almost_equal((Q.T*Q).todense(),eye(Q.shape[1])) - - for fine,coarse in zip(fine_candidates,coarse_candidates): - assert_almost_equal(fine,Q*coarse) - - #aggregate one more level (to a single aggregate) - K = len(coarse_candidates) - N = K*AggOp.shape[1] - AggOp = csr_matrix((ones(N),zeros(N),arange(N + 1)),dims=(N,1)) - fine_candidates = coarse_candidates - - Q,coarse_candidates = fit_candidates(AggOp,fine_candidates) - - assert_equal(len(coarse_candidates),len(fine_candidates)) - assert_almost_equal((Q.T*Q).todense(),eye(Q.shape[1])) - - for fine,coarse in zip(fine_candidates,coarse_candidates): - assert_almost_equal(fine,Q*coarse) - + pass + if __name__ == '__main__': NumpyTest().run() Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-03 21:37:54 UTC (rev 3395) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-03 22:43:43 UTC (rev 3396) @@ -1,6 +1,6 @@ from numpy.testing import * -from numpy import sqrt,empty,ones,arange,array_split +from numpy import sqrt,empty,ones,arange,array_split,eye,array,zeros,diag from scipy import rand from scipy.sparse import spdiags,csr_matrix,lil_matrix import numpy @@ -66,6 +66,8 @@ assert_array_equal(S_result.todense(),S_expected.todense()) + + # note that this method only tests the current implementation, not # all possible implementations def reference_sa_constant_interpolation(A,epsilon): @@ -155,6 +157,59 @@ ## S_expected = reference_sa_constant_interpolation(A,epsilon) ## assert_array_equal((S_result - S_expected).nnz,0) +class TestFitCandidates(NumpyTestCase): + def setUp(self): + self.normal_cases = [] + + #one candidate + self.normal_cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)),[ones(5)])) + self.normal_cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)),[ones(5)])) + self.normal_cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9)])) + self.normal_cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)),[arange(9)])) + + #two candidates + self.normal_cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)),[ones(4),arange(4)])) + self.normal_cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9),arange(9)])) + self.normal_cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)),[ones(9),arange(9)])) + + #block candidates + self.normal_cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[array([1]*9 + [0]*9),arange(2*9)])) + + #TODO add test case where aggregation operator has holes + + def check_normal(self): + """Test case where aggregation includes all fine nodes""" + + for AggOp,fine_candidates in self.normal_cases: + Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) + + assert_equal(len(coarse_candidates),len(fine_candidates)) + + #each fine level candidate should be fit exactly + for fine,coarse in zip(fine_candidates,coarse_candidates): + assert_almost_equal(fine,Q*coarse) + assert_almost_equal(Q*(Q.T*fine),fine) + + + #aggregate one more level (to a single aggregate) + K = len(coarse_candidates) + N = K*AggOp.shape[1] + AggOp = csr_matrix((ones(N),zeros(N),arange(N + 1)),dims=(N,1)) #aggregate to a single point + fine_candidates = coarse_candidates + + #now check the coarser problem + Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) + + assert_equal(len(coarse_candidates),len(fine_candidates)) + + for fine,coarse in zip(fine_candidates,coarse_candidates): + assert_almost_equal(fine,Q*coarse) + assert_almost_equal(Q*(Q.T*fine),fine) + + + + + if __name__ == '__main__': NumpyTest().run() From scipy-svn at scipy.org Wed Oct 3 19:58:36 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 18:58:36 -0500 (CDT) Subject: [Scipy-svn] r3397 - in trunk/scipy/io: . nifti nifti/bin nifti/man nifti/nifti nifti/tests nifti/tests/data Message-ID: <20071003235836.79A2F39C0A1@new.scipy.org> Author: chris.burns Date: 2007-10-03 18:58:26 -0500 (Wed, 03 Oct 2007) New Revision: 3397 Added: trunk/scipy/io/nifti/ trunk/scipy/io/nifti/AUTHOR trunk/scipy/io/nifti/COPYING trunk/scipy/io/nifti/Changelog trunk/scipy/io/nifti/MANIFEST.in trunk/scipy/io/nifti/Makefile trunk/scipy/io/nifti/PKG-INFO trunk/scipy/io/nifti/README.html trunk/scipy/io/nifti/TODO trunk/scipy/io/nifti/bin/ trunk/scipy/io/nifti/bin/pynifti_pst trunk/scipy/io/nifti/man/ trunk/scipy/io/nifti/man/pynifti_pst.1 trunk/scipy/io/nifti/nifti/ trunk/scipy/io/nifti/nifti/__init__.py trunk/scipy/io/nifti/nifti/nifticlib.i trunk/scipy/io/nifti/nifti/nifticlib.py trunk/scipy/io/nifti/nifti/niftiimage.py trunk/scipy/io/nifti/nifti/utils.py trunk/scipy/io/nifti/setup.py trunk/scipy/io/nifti/tests/ trunk/scipy/io/nifti/tests/data/ trunk/scipy/io/nifti/tests/data/example4d.nii.gz trunk/scipy/io/nifti/tests/test_fileio.py trunk/scipy/io/nifti/tests/test_main.py trunk/scipy/io/nifti/tests/test_utils.py Log: Check in current version of PyNifti which has an MIT License. Added: trunk/scipy/io/nifti/AUTHOR =================================================================== --- trunk/scipy/io/nifti/AUTHOR 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/AUTHOR 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,2 @@ +Michael Hanke + Added: trunk/scipy/io/nifti/COPYING =================================================================== --- trunk/scipy/io/nifti/COPYING 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/COPYING 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2006-2007 Michael Hanke + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. Added: trunk/scipy/io/nifti/Changelog =================================================================== --- trunk/scipy/io/nifti/Changelog 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/Changelog 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,137 @@ +pynifti (0.20070930.1-1) unstable; urgency=low + + * Relicense under the MIT license, to be compatible with SciPy license. + http://www.opensource.org/licenses/mit-license.php + * Updated documentation. + + -- Michael Hanke Sun, 30 Sep 2007 13:24:15 +0200 + +pynifti (0.20070917.1-1) unstable; urgency=low + + [ Michael Hanke ] + * Bugfix: Can now update NIfTI header data when no filename is set + (Closes: #442175). + * Unloading of image data without a filename set is no checked and prevented + as it would damage data integrity and the image data could not be + recovered. + + [ Yaroslav Halchenko ] + * Added 'pixdim' property. + + -- Michael Hanke Mon, 17 Sep 2007 08:32:59 +0200 + +pynifti (0.20070905.1-1) unstable; urgency=low + + [ Yaroslav Halchenko ] + * Fixed a bug in the qform/quaternion handling that caused changes to the + qform to vanish when saving to file. + + [ Michael Hanke ] + * Added more unit tests. + * 'dim' vector in the NIfTI header is now guaranteed to only contain + non-zero elements. This caused problems with some applications. + + -- Michael Hanke Wed, 5 Sep 2007 20:54:18 +0200 + +pynifti (0.20070803.1-1) unstable; urgency=low + + * Does not depend on SciPy anymore. + * Initial steps towards a unittest suite. + * pynifti_pst can now print the peristimulus signal matrix for a single + voxel (onsets x time) for easier processing of this information in + external applications. + * utils.getPeristimulusTimeseries() can now be used to compute mean and + variance of the signal (among others). + * debian/{copyright|watch} now point to the new home of pynifti in the + niftilib project on SourceForge. + * pynifti_pst is able to compute more than just the mean peristimulus + timeseries (e.g. variance and standard deviation). + * Relaxed build-dependency on libniftiio to any SO-version of the library. + * Set default image description when saving a file if none is present. + * Improved documentation. + + -- Michael Hanke Fri, 3 Aug 2007 15:47:26 +0200 + +pynifti (0.20070425.1-1) unstable; urgency=low + + * Improved documentation. Added note about the special usage of the header + property. Also added notes about the relevant properties in the docstring + of the corresponding accessor methods. + * Added property and accessor methods to access/modify the repetition time + of timeseries (dt). + * Added functions to manipulate the pixdim values. + * Added utils.py with some utility functions. + * Added functions/property to determine the bounding box of an image. + * Fixed a bug that caused a corrupted sform matrix when converting a NumPy + array and a header dictionary into a NIfTI image. + * Added script to compute peristimulus timeseries (pynifti_pst). + * Package now depends on python-scipy. + + -- Michael Hanke Wed, 25 Apr 2007 23:12:22 +0200 + +pynifti (0.20070315.1-1) unstable; urgency=low + + [ Yaroslav Halchenko ] + * Removed functionality for "NiftiImage.save() raises an IOError + exception when writing the image file fails." + + [ Michael Hanke ] + * Added ability to force a filetype when setting the filename or saving + a file. + * Reverse the order of the 'header' and 'load' argument in the NiftiImage + constructor. 'header' is now first as it seems to be used more often. + * Improved the source code documentation. + * Added getScaledData() method to NiftiImage that returns a copy of the data + array that is scaled with the slope and intercept stored in the NIfTI + header. + + -- Michael Hanke Thu, 15 Mar 2007 18:25:52 +0100 + +pynifti (0.20070301.2-1) unstable; urgency=low + + * Fixed wrong link to the source tarball in README.html. + + -- Michael Hanke Thu, 1 Mar 2007 22:27:26 +0100 + +pynifti (0.20070301.1-1) unstable; urgency=low + + [ Michael Hanke ] + * Updated build-depends to comply to the latest Python policy. + * Initial upload to the Debian archive. (Closes: #413049) + * NiftiImage.save() raises an IOError exception when writing the image file + fails. + + [ Yaroslav Halchenko ] + * Added extent, volextent, and timepoints properties to NiftiImage + class. + + -- Michael Hanke Thu, 1 Mar 2007 22:08:15 +0100 + +pynifti (0.20070220.1-1) unstable; urgency=low + + * NiftiFile class is renamed to NiftiImage. + * SWIG-wrapped libniftiio functions are no available in the nifticlib + module. + * Fixed broken NiftiImage from Numpy array constructor. + * Added initial documentation in README.html. + * Fulfilled a number of Yarik's wishes ;) + + -- Michael Hanke Tue, 20 Feb 2007 17:36:08 +0100 + +pynifti (0.20070214.1-1) unstable; urgency=low + + * Does not depend on libfslio anymore. + * Up to seven-dimensional dataset are supported (as much as NIfTI can do). + * The complete NIfTI header dataset is modifiable. + * Most image properties are accessable via class attributes and accessor + methods. + * Improved documentation (but still a long way to go). + + -- Michael Hanke Wed, 14 Feb 2007 10:11:55 +0100 + +pynifti (0.20061114-1) unstable; urgency=low + + * Initial release. + + -- Michael Hanke Tue, 14 Nov 2006 19:36:51 +0100 + Added: trunk/scipy/io/nifti/MANIFEST.in =================================================================== --- trunk/scipy/io/nifti/MANIFEST.in 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/MANIFEST.in 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,5 @@ +include AUTHOR COPYING Makefile README.html MANIFEST.in +include Changelog TODO +include man/* +include tests/* tests/data/* +prune nifti/wrap.py Added: trunk/scipy/io/nifti/Makefile =================================================================== --- trunk/scipy/io/nifti/Makefile 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/Makefile 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,43 @@ +all: + +distclean: + -rm MANIFEST Changelog + -rm nifti/*.{c,pyc,so} nifti/nifticlib.py + -rm tests/*.pyc + -rm -r build + -rm -r dist + + +orig-src: distclean + # clean existing dist dir first to have a single source tarball to process + -rm -rf dist + # the debian changelog is also the upstream changelog + cp debian/changelog Changelog + + # update manpages + help2man -N -n "compute peristimulus timeseries of fMRI data" \ + bin/pynifti_pst > man/pynifti_pst.1 + + if [ ! "$$(dpkg-parsechangelog | egrep ^Version | cut -d ' ' -f 2,2 | cut -d '-' -f 1,1)" == "$$(python setup.py -V)" ]; then \ + printf "WARNING: Changelog version does not match tarball version!\n" ;\ + exit 1; \ + fi + # let python create the source tarball + python setup.py sdist --formats=gztar + # rename to proper Debian orig source tarball and move upwards + # to keep it out of the Debian diff + file=$$(ls -1 dist); ver=$${file%*.tar.gz}; ver=$${ver#pynifti-*}; mv dist/$$file ../pynifti_$$ver.orig.tar.gz + +bdist_wininst: + # THIS IS ONLY FOR WINDOWS! + # Consider this a set of notes on how to build PyNIfTI on win32, rather + # than an actually working target + # + # assumes Dev-Cpp to be installed at C:\Dev-Cpp + python setup.py build_ext -c mingw32 --swig-opts "-C:\Dev-Cpp\include/nifti -DWIN32" -IC:\Dev-Cpp\include nifti + + # for some stupid reason the swig wrapper is in the wrong location + move /Y nifticlib.py nifti + + # now build the installer + python setup.py bdist_wininst Added: trunk/scipy/io/nifti/PKG-INFO =================================================================== --- trunk/scipy/io/nifti/PKG-INFO 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/PKG-INFO 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: pynifti +Version: 0.20070930.1 +Summary: Python interface for the NIfTI IO libraries +Home-page: http://apsy.gse.uni-magdeburg.de/hanke +Author: Michael Hanke +Author-email: michael.hanke at gmail.com +License: MIT License +Description: +Platform: UNKNOWN Added: trunk/scipy/io/nifti/README.html =================================================================== --- trunk/scipy/io/nifti/README.html 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/README.html 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,391 @@ + + + + + + PyNIfTI - Python bindings to NIfTI + + + + + + +

PyNIfTI - Python-style access to NIfTI and ANALYZE files

+ + +

1. What is NIfTI and what do I need PyNIfTI for?

+ +

NIfTI

+

NIfTI is a new Analyze-style data +format, proposed by the +NIfTI Data Format Working Group +as a "short-term measure to facilitate inter-operation of functional MRI data +analysis software packages".

+

Meanwhile a number of toolkits are NIfTI-aware (e.g. FSL, AFNI, SPM, +Freesurfer and a to a certain degree also Brainvoyager). +Additionally, dicomnifti +allows the direct conversion from DICOM images into the NIfTI format.

+

With libniftiio +there is a reference implementation of a C library to read, write and manipulate +NIfTI images. The library source code is put into the public domain and a +corresponding project is hosted at +SourceForge.

+

In addition to the C library, there is also an IO library written in Java and +Matlab functions to make use of NIfTI files from within Matlab.

+ + +

Python

+

Unfortunately, it is not that trivial to read NIfTI images with Python. This is +particularly sad, because there is a large number of easy-to-use, high-quality +libraries for signal processing available for Python (e.g. SciPy).

+

Moreover Python has bindings to almost any important language/program +in the fields of maths, statistics and/or engineering. If you want to use +R to calculate +some stats in a Python script, simply use RPy +and pass any data to R. If you don't care about R, but Matlab is your one and +only friend, there are at least two different Python modules to control Matlab +from within Python scripts. Python is the glue between all those helpers and the Python user is +able to combine as many tools as necessary to solve a given problem +-- the easiest way.

+ + +

PyNIfTI

+

PyNIfTI aims to provide easy access to NIfTI images from within Python. It +uses SWIG-generated wrappers for the NIfTI +reference library and provides the NiftiImage class for +Python-style access to the image data.

+

While PyNIfTI is not yet complete (i.e. doesn't support everything the +C library can do), it already provides access to the most +important features of the NIfTI-1 data format and libniftiio +capabilities. The following features are currently implemented:

+
    + +
  • PyNIfTI can read and write any file format supported by libniftiio. This +includes NIfTI (single and pairs) as well as ANALYZE files.
    • +
  • PyNIfTI provides fast and convenient access to the image data via +NumPy arrays. This should enable users to +process image data with most (if not all) numerical routines available for +Python. The NumPy array automatically uses a datatype corresponding to the NIfTI image data -- no +unnecessary upcasting is performed.
    • +
  • PyNIfTI provides full read and write access to the NIfTI header data. Header information can +be exported to a Python dictionary and can also be updated by using information +from a dictionary.
    • +
  • Instead of accessing NIfTI data from files, PyNIfTI is able to create NIfTI +images from NumPy arrays. The appropriate NIfTI header information is determined +from the array properties. Additional header information can be optionally +specified -- making it easy to clone NIfTI images if necessary, but with minor +modifications.
    • +
  • Most properties of NIfTI images are accessible via attributes and/or accessor +functions of the NiftiImage. Inter-dependent properties are +automatically updated if necessary (e.g. modifying the Q-Form matrix also updates +the pixdim properties and quaternion representation).
    • +
  • All properties are accessible via Python-style datatypes: A 4x4 matrix is +an array not 16 individual numbers.
    • +
  • PyNIfTI should be resonably fast. Image data will only be loaded into the memory +if necessary. Simply opening a NIfTI file to access some header data is +performed with virtually no delay independent of the size of the image. Unless image +resizing or datatype conversion must be performed the image data can be +shared by the NIfTI image and accessing NumPy arrays, and therefore memory won't be wasted +memory with redundant copies of the image data. However, one should be careful to make a +copy of the image data if you intend to resize and cast the image data (see the +docstring of the NiftiImage.asarray() method).
    • +
+ +

Scripts

+

Some functions provided by PyNIfTI also might be useful outside the Python +environment. Therefore I plan to add some command line scripts to the package.

+

Currently there is only one: pynifti_pst (pst: peristimulus +timecourse). Using this script one can compute the signal timecourse for a +certain condition for all voxels in a volume at once. This might be useful for +exploring a dataset and accompanies similar tools like FSL's tsplot.

+

The output of pynifti_pst can be loaded into FSLView to simultaneously +look at statistics and signal timecourses. Please see the corresponding example below.

+ +

Known issues aka bugs

+
    +
  • PyNIfTI currently ignores the origin field of ANALYZE files - it is neither read + nor written. A possible workaround is to convert ANALYZE files into the NIfTI format + using FSL's avwchfiletype.
    • +
+ + +

2. License

+

PyNIfTI is written by Michael Hanke +as free software (both beer and speech) and licensed under the +MIT License. +

+ + +

3. Download

+

As PyNIfTI is still pretty young, a number of significant +improvements/modifications are very likely to happen in the near future. If you +discover any bugs or you are missing some features, please be sure to check the +SVN repository (read below) if your problem is already solved.

+ +

+

Source code

+

Since June 2007 PyNIfTI is part of the +niftilibs family. The PyNIfTI +source code can be obtained from the +Sourceforge project site. +

+ +

Binary packages

+

GNU/Linux

+

PyNIfTI is available in recent versions of the Debian (since lenny) and +Ubuntu (since gutsy in universe) distributions. The name of the binary package +is python-nifti in both cases.

+ +

Binary packages for some additional Debian and (K)Ubuntu versions are also +available. Please visit +this page to read +about how you have to setup your system to retrieve the PyNIfTI package via +your package manager and stay in sync with future releases.

+ +

Windows

+

A binary installer for a recent Python version is available from the +Sourceforge project site. + + +

Macintosh

+

Unfortunately, no binary packages are available. I have no access to such +a machine at the moment. But it is possible to build PyNIfTI from source on +Mac OS X (see below for more information).

+ + + +

4. Installation

+ +

Compile from source: General instructions

+

PyNIfTI needs a few things to build and run properly:

+ +

Make sure that the compiled nifticlibs and the corresponding headers are +available to your compiler. If they are located in a custom directory, you +might have to specify --include-dirs and +--library-dirs options to the build command below.

+

Once you have downloaded the sources, extract the tarball and enter the root +directory of the extracted sources. A simple

+

python setup.py build_ext

+

should build the SWIG wrappers. If this has been done +successfully, all you need to do is install the modules by invoking

+

sudo python setup.py install

+

If sudo is not configured (or even installed) you might have to use +su instead.

+

Now fire up Python and try importing the module to see if everything is +fine. It should look similar to this:

+
+Python 2.4.4 (#2, Oct 20 2006, 00:23:25)
+[GCC 4.1.2 20061015 (prerelease) (Debian 4.1.1-16.1)] on linux2
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import nifti
+>>> 
+
+

Windows

+

It should be pretty straightforward to compile PyNIfTI for win32. The most +convenient way seems to be using the +Dev-Cpp IDE +and the DevPak of the nifticlibs. Have a look into the toplevel Makefile of the +PyNIfTI source distribution for some hints.

+ + +

MacOS X and MacPython

+

When you are comiling PyNIfTI on MacOS X and want to use it with MacPython, +please make sure that the NIfTI C libraries are compiled as fat binaries +(compiled for both ppc and i386). Otherwise +PyNIfTI extensions will not compile.

+

One can achieve this by adding both architectures to the CFLAGS +definition in the toplevel Makefile of the NIfTI C library source code. Like +this

+

CFLAGS = $(ANSI_FLAGS) -arch ppc -arch i386

+ + +

Binary packages

+

GNU/Linux

+

If you have configured your system as described on +this page all you +have to do to install PyNIfTI is this:

+

apt-get update
apt-get install python-nifti

+

This should pull all necessary dependencies. If it doesn't, it's a bug that +should be reported.

+

Windows

+

As always: click Next as long as necessary and finally Finish. + +

Troubleshooting

+

If you get an error when importing the nifti module in Python +complaining about missing symbols your niftiio library contains references to +some unresolved symbols. Try adding znzlib and zlib to the +linker options the PyNIfTI setup.py, like this:

+ +

libraries = [ 'niftiio', 'znz', 'z' ],

+ +

5. Things to know

+

When accessing NIfTI image data through NumPy arrays the order of the +dimensions is reversed. If the x, y, z, t dimensions of a NIfTI image +are 64, 64, 32, 456 (as for example reported by nifti_tool), the shape +of the NumPy array (e.g. as returned by NiftiImage.asarray()) will +be: 456, 32, 64, 64.

+

This is done to be able to slice the data array much easier in the most +common cases. For example, if you are interested in a certain volume of a timeseries +it is much easier to write data[2] instead of +data[:,:,:,2], right?. + +

6. Examples

+

The next sections contains some examples showing ways to use PyNIfTI to +read and write imaging data from within Python to be able to process it with +some random Python library.

+

All examples assume that you have imported the PyNIfTI module by invoking:

+

from nifti import *

+ +

a) Fileformat conversion

+

Open the MNI standard space template that is shipped with FSL. No filename +extension is necessary as libniftiio determines it automatically:

+ +

nim = NiftiImage('avg152T1_brain')

+ +

The filename is available via the 'filename' attribute:

+

print nim.filename

+

yields 'avg152T1_brain.img'. This indicates an ANALYZE image. If you want to +save this image as a single gzipped NIfTI file simply do:

+

nim.save('mni.nii.gz')

+

The filetype is determined from the filename. If you want to save to gzipped +ANALYZE file pairs instead the following would be an alternative to calling the +save() with a new filename.

+

nim.filename = 'mni_analyze.img.gz'
nim.save()

+

Please see the docstring of the NiftiImage.setFilename() method +to learn how the filetypes are determined from the filenames.

+ +

b) NIfTI files from array data

+

The next code snipped demonstrates how to create a 4d NIfTI image containing +gaussian noise. First we need to import the NumPy module

+

import numpy

+

Now generate the noise dataset. Let's generate noise for 100 volumes with 16 +slices and a 32x32 inplane matrix.

+

noise = numpy.random.randn(100,16,32,32)

+

Please notice the order in which the dimensions are specified: +(t, z, y, x).

+

The datatype of the array will most likely be float64 -- which can +be verified by invoking noise.dtype.

+

Converting this dataset into a NIfTI image is done by invoking the +NiftiImage constructor with the noise dataset as argument:

+

nim = NiftiImage(noise)

+ +

The relevant header information is extracted from the NumPy array. If you +query the header information about the dimensionality of the image, it returns +the desired values:

+

print nim.header['dim'] # yields: [4, 32, 32, 16, 100, 0, 0, 0]

+First value shows the number of dimensions in the datset: 4 (good, that's what +we wanted). The following numbers are dataset size on the x, y, z, t, u, v, w +axis (NIfTI files can handle up to 7 dimensions). Please notice, that the order +of dimensions is now 'correct': We have 32x32 inplane resolution, 16 slices in z +direction and 100 volumes.

+

Also the datatype was set appropriately. The exprression:

+

nim.header['datatype'] == nifticlib.NIFTI_TYPE_FLOAT64

+

will evaluate to True.

+

To save the noise file to disk, just call the save() method:

+

nim.save('noise.nii.gz')

+ + +

c) Select ROIs

+

Suppose you want to have the first ten volumes of the noise dataset we have +just created in a separate file. First open the file (can be skipped if it is +still open):

+

nim = NiftiImage('noise.nii.gz')

+

Now select the first ten volumes and store them to another file, while +preserving as much header information as possible:

+

+nim2 = NiftiImage(nim.data[:10], nim.header)
+nim2.save('part.hdr.gz') +

+

The NiftiImage constructor takes a dictionary with header information as an +optional argument. Settings that are not determined by the array (e.g. size, +datatype) are copied from the dictionary and stored to the new NIfTI image.

+ + +

d) Linear detrending of timeseries (SciPy module is required for this +example)

+

Let's load another 4d NIfTI file and perform a linear detrending, by fitting +a straight line to the timeseries of each voxel and substract that fit from the +data. Although this might sound complicated at first, thanks to the excellent +SciPy module it is just a few lines of code.

+

nim = NiftiImage('timeseries.nii')

+

Depending on the datatype of the input image the detrending process might +change the datatype from integer to float. As operations that change the +(binary) size of the NIfTI image are not supported, we need to make a copy +of the data and later create a new NIfTI image.

+

data = nim.asarray()

+

Now detrend the data along the time axis. Remember that the array has the +time axis as its first dimension (in contrast to the NIfTI file where it is +the 4th).

+

+from scipy import signal
+data_detrended = signal.detrend( data, axis=0 )

+

Finally, create a new NIfTI image using header information from the original +source image.

+

nim_detrended = NiftiImage( data_detrended, nim.header)

+ +

e) Make a quick plot of a voxels timeseries (Gnuplot module is required)

+

Plotting is essential to get a 'feeling' for the data. The +python interface +to Gnuplot makes it really easy to plot +something (e.g. when running Python +interactively via IPython). Please +note, that there are many other possibilities for plotting. Some examples are: +using R via +RPy or Matlab-style plotting via +matplotlib.

+

However, using Gnuplot is really easy. First import the Gnuplot module +and create the interface object.

+

+from Gnuplot import Gnuplot
+gp = Gnuplot()

+

We want the timeseries as a line plot and not just the datapoints, so +let's talk with Gnuplot.

+

gp('set data style lines')

+

Now load a 4d NIfTI image

+

nim = NiftiImage('perfect_subject.nii.gz')

+

and finally plot the timeseries of voxel (x=20, y=30, z=12):

+

gp.plot(nim.data[:,12,30,20])

+

A Gnuplot window showing the timeseries should popup now (screenshot). Please read the +Gnuplot Manual to learn what it can do -- and it can do a lot more than just +simple line plots (have a look at this +page if you are interested).

+ +

f) Show a slice of a 3d volume (Matplotlib module is required)

+

This example demonstrates howto use the Matlab-style plotting of Matplotlib to view a slice from a 3d volume.

+

This time I assume that a 3d nifti file is already opened and available in the nim3d object. At first we need to load the necessary Python module.

+

from pylab import *

+

If everything went fine, we can now view a slice (x,y):

+

imshow(nim3d.data[200], interpolation='nearest', cmap=cm.gray)
+show()

+

It is necessary to call the show() function one time after importing pylab to actually see the image when running Python interactively (screenshot).

When you want to have a look at a yz-slice, NumPy array magic comes into play.

+

imshow(nim3d.data[::-1,:,100], interpolation='nearest', cmap=cm.gray)

+

The ::-1 notation causes the z-axis to be flipped in the images. This makes a much nicer screenshot, because the used example volume has the z-axis originally oriented upsidedown.

+ +

g) Compute and display peristimulus signal timecourse of multiple conditions with pynifti_pst and FSLView

+

Sometimes one wants to look at the signal timecourse of some voxel after a certain stimulation onset. An easy way would be to have some fMRI data viewer that displays a statistical map and one could click on some activated voxel and the peristimulus signal timecourse of some condition in that voxel would be displayed.

+

This can easily be done by using pynifti_pst and FSLView.

+

pynifti_pst comes with a manpage that explains all options and arguments. Basically pynifti_pst need a 4d image (e.g. an fMRI timeseries; possibly preprocessed/filtered) and some stimulus onset information. This information can either be given directly on the command line or is read from files. Additionally one can specify onsets as volume numbers or as onset times.

+

pynifti_pst understands the FSL custom EV file format so one can easily use those files as input.

+

An example call could look like this:

+

pynifti_pst --times --nvols 5 -p uf92.feat/filtered_func_data.nii.gz pst_cond_a.nii.gz uf92.feat/custom_timing_files/ev1.txt uf92.feat/custom_timing_files/ev2.txt

+

This computes a peristimulus timeseries using the preprocessed fMRI from a FEAT output directory +and two custom EV files that both together make up condition A. --times indicates that +the EV files list onset times (not volume ids) and --nvols requests the mean peristimulus +timecourse for 4 volumes after stimulus onset (5 including onset). -p recodes the +peristimulus timeseries into percent signalchange, where the onset is always zero and any following +value is the signal change with respect to the onset volume.

+

This call produces a simple 4d NIfTI image that can be loaded into FSLView as any other timeseries. The following call can be used to display an FSL zmap from the above results path on top of some anatomy. Additionally the peristimulus timeseries of two conditions are loaded. This screenshot shows how it could look like. One of the nice features of FSLView is that its timeseries window can remember selected curves, which can be useful to compare signal timecourses from different voxels (blue and green line in the screenshot).

+

fslview pst_cond_a.nii.gz pst_cond_b.nii.gz uf92_ana.nii.gz uf92.feat/stats/zstat1.nii.gz -b 3,5

+ +

History

+

The full changelog is here.

+ + Added: trunk/scipy/io/nifti/TODO =================================================================== --- trunk/scipy/io/nifti/TODO 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/TODO 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,55 @@ + +Yarik's wishes sorted - please remove or fulfill: +------------------------------------------------- + +- as it is now: getHeader returns a header dictionary which is not + binded to the object. so direct manipulation to it such as + + inNifti.header['bla']='bleh' + are lost and the only way to introduce a change to the field seems + to operate as + h = inNifti.header + h['bla'] = 'bleh' + inNifti.header = h + which is at least should be documented in capital letters! ;-) + + Proposed solution: + + introduce private __header, which would result in singleton behavior + of getHeader -- on first call it does all the querying from + self.__nimg and nhdr2dict, + but on consequtive calls - simply return __header. + + may be nhdr (result of _nim2nhdr) should be also stored as __nhdr + to avoid consequtive invocations of a costly function + + Then updateHeader should + * allow not specified hdrdict (default to None) + * not query __nimg if __header/__nhdr is not None + and simply use nhdr=self.__nhdr + * if hdrdict=None (ie not specified) make a copy + hdrdict == self.__header.copy(), so del commands + dont' do evil things, and proceed further ;-) + + save method also should invoke updateHeader I guess so that if + there were any changes to the header dictionary, they get saved.. + + Michael's comment: + ------------------ + Although this would be nice to have, I really think it could be the + source of some ugly bugs. Maintaining a separate copy of the header means + that PyNifit has to keep track of all possible dependencies between the + image properties by itself, instead of relying on the nifticlib to do. + this. I'd prefer to implement as much accessor methods or properties as + necessary to make calls to the header property superflous. + + BTW: The same behavior is true for the qform and sform properties + (including their inverse versions). + + +Possibly wrong/incorrect/wontfix suggestions: +--------------------------------------------- + +- [gs]etVoxDims should work with all dimensions up to 7 as they are + defined in nifti1_io.h + hanke: maybe, but that would mix different units, which I'd rather not do. Added: trunk/scipy/io/nifti/bin/pynifti_pst =================================================================== --- trunk/scipy/io/nifti/bin/pynifti_pst 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/bin/pynifti_pst 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,251 @@ +#!/usr/bin/python +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# Copyright (C) 2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + + +import os, sys +from optparse import OptionParser +import nifti.utils +import numpy as np + +def loadEVFile( filename ): + try: + file = open( filename ) + except IOError: + raise ValueError, "Cannot open EV file '%s'" % filename + + onsets = [] + + # split each line and ignore empty lines + for line in file: + e = line.split() + if len(e): + onsets.append( float( e[0] ) ) + + return onsets + + +def checkCmdLine(args, opts, min_args): + if len(args) < min_args: + raise RuntimeError, \ + "%s: error: Incorrect number of arguments " \ + "(supplied %i, needed %i)." % (sys.argv[0], len(args), min_args) + + +def main(): + parser = OptionParser( \ + usage="%prog [options] <4dimage> [...]", \ + version="%prog 0.20070424", description="""\ +%prog computes the peristimulus signal timecourse for all voxels in a volume at +once. Stimulus onsets can be specified as volume numbers or times (will be +converted into volume numbers using a supplied repetition time). Onsets can be +specified directly on the command line, but can also be read from (multiple) +files. Such file are assumed to list one onset per line (as the first value). +Empty lines are ignored. This enables %prog to use e.g. FSL's custom EV files. +If several files are specified, the read onsets are combined to a single onset +vector. +%prog writes a 4d timeseries image as output. This image can e.g. be loaded into +FSLView to look at each voxels signal timecourse in a certain condition by +simply clicking on it. +""" ) + + # define options + parser.add_option('--verbose', action='store_true', dest='verbose', + default=False, help='print status messages') + parser.add_option('-n', '--nvols', action='store', dest='nvols', + default=10, type="int", help="""\ +Set the length of the computed peristimulus signal timecourse (in volumes). +Default: 10 +""" ) + parser.add_option('-t', '--times', action='store_true', dest='times', + default=False, help="""\ +If supplied, the read values are treated as onset times and will be converted +to volume numbers. For each onset the volume that is closest in time will be +selected. Volumes are assumed to be recorded exactly (and completely) after +tr/2, e.g. if 'tr' is 2 secs the first volume is recorded at exactly one +second. Please see the --tr and --offset options to learn how to adjust the +conversion. """ ) + parser.add_option('--tr', action='store', dest='tr', default=None, + type="float", help="""\ +Repetition time of the 4d image (temporal difference of two successive +volumes). This can be used to override the setting in the 4d image. The +repetition time is necessary to convert onset times into volume numbers. +If the '--times' option is not set this value has no effect. Please note +that repetitions time and the supplied onsets have to be in the same unit. + +Please note, that if --times is given the tr has to be specified in the +same unit as the read onset times. +""" ) + parser.add_option('--offset', dest='offset', action='store', default=0.0, + type='float', help="""\ +Constant offset applied to the onsets times when converting them to volume +numbers. Without setting '--times' this option has no effect'. +""" ) + parser.add_option('-p', '--percsigchg', action='store_true', + dest='perc_sig_chg', default=False, help="""\ +Convert the computed timecourse to percent signal change relative to the first +(onset) volume. This might not be meaningful when --operation is set to +something different than 'mean'. Please note, that the shape of the computes +timeseries heavily depends on the first average volume. It might be more +meaningful to use a real baseline condition as origin. However, this is not +supported yet. Default: False +""" ) + parser.add_option('--printvoxel', action='store', dest='printvoxel', + default=None, help="""\ +Print the peristimulus timeseries of a single voxel for all onsets separately. +This will print a matrix (onsets x time), where the number of columns is +identical to the value of --nvols and the number of rows corresponds to the +number of specified onsets. (e.g. 'z,y,x') +""" ) + parser.add_option('--operation', action='store', dest='operation', + default='mean', help="""\ +Choose the math operation that is performed to compute the peristimulus +timeseries. By default this is the mean across all stimulations ('mean'). +Other possibilities are the standard deviation ('std') and standard error +('sde'). +""" ) + # parse options + (opts, args) = parser.parse_args() # read sys.argv[1:] by default + + try: + checkCmdLine( args, opts, 3 ) + except: + print sys.exc_info()[1] + sys.exit( 1 ) + + if opts.verbose: + print "Read 4d image '%s'" % args[0] + nimg = nifti.NiftiImage( args[0] ) + + if not nimg.timepoints > 1: + print "%s: error: Need 4d image as input. " \ + "Supplied image only has one volume." \ + % sys.argv[0] + sys.exit( 1 ) + + # determine the requested function for timeseries calculation + if opts.operation == 'mean': + pst_fx = np.mean + elif opts.operation == 'std' or opts.operation == 'sde': + pst_fx = np.std + else: + print "'%s' is not a supported operation." % opts.operation + sys.exit(1) + + outfilename = args[1] + if opts.verbose: + print "Output will be written to '%s'" % outfilename + + if opts.tr: + tr = opts.tr + if opts.verbose: + print "Using provide repetition time: %f" % tr + else: + tr = nimg.rtime + if opts.verbose: + print "Using repetition from 4d image: %f" % tr + + onsets = [] + + for src in args[2:]: + if os.path.isfile( src ): + if opts.verbose: + print "Reading values from '%s'" % src + onsets += loadEVFile( src ) + else: + try: + onsets.append( float( src ) ) + except ValueError: + print "%s: error: '%s' cannot be converted into a " \ + "floating-point value" % (sys.argv[0], src) + sys.exit( 1 ) + + if opts.times: + if opts.verbose: + print "Convert supplied onset times into volumes " \ + "(tr: %f, offset: %f)" % (tr, opts.offset) + + onsetvols = nifti.utils.time2vol( onsets, + tr, + opts.offset, + decimals = 0 ).astype('int') + else: + if opts.verbose: + print "Verify onset volume numbers" + onsetvols = [ int(i) for i in onsets ] + + if opts.verbose: + print "Selected volumes (index starts at zero!):\n" + ', '.join([ str(o) for o in onsetvols]) + + if opts.verbose: + print "Compute mean peristimulus signal timecourse " \ + "(length: %i volumes)" % opts.nvols + + if opts.printvoxel: + # get timeseries for each onset + pst = nifti.utils.getPeristimulusTimeseries( nimg, + onsetvols, + opts.nvols, + tuple ).copy() + # make matrix ( onset x time ) + voxel_pst = eval('pst[:,:,' + opts.printvoxel +'].T') + + for onset in voxel_pst: + for t in onset: + print t, + print '' + + + if opts.verbose: + print "Compute peristimulus timeseries" + + pst = nifti.utils.getPeristimulusTimeseries( nimg, + onsetvols, + opts.nvols, + pst_fx ).copy() + + # divide by srqt of number of stimulations if standard error is requested + if opts.operation == 'sde': + pst /= np.sqrt( len(onsetvols) ) + + if opts.perc_sig_chg: + if opts.verbose: + print "Convert to percent signal change" + baseline = nifti.utils.getPeristimulusTimeseries( nimg, + onsetvols, + 1, + np.mean )[0].copy() + + if opts.operation == 'mean': + pst -= baseline + + # only divide if baseline is different from zero + pst[:,baseline != 0] /= baseline[baseline != 0] + pst *= 100.0 + + if opts.verbose: + print "Write output" + onimg = nifti.NiftiImage(pst,nimg.header) + onimg.save( outfilename ) + + if opts.verbose: + print "Done" + + +if __name__ == "__main__": + main() + + + Property changes on: trunk/scipy/io/nifti/bin/pynifti_pst ___________________________________________________________________ Name: svn:executable + * Added: trunk/scipy/io/nifti/man/pynifti_pst.1 =================================================================== --- trunk/scipy/io/nifti/man/pynifti_pst.1 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/man/pynifti_pst.1 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,82 @@ +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.36. +.TH PYNIFTI_PST "1" "September 2007" "pynifti_pst 0.20070424" "User Commands" +.SH NAME +pynifti_pst \- compute peristimulus timeseries of fMRI data +.SH DESCRIPTION +usage: pynifti_pst [options] <4dimage> [...] +.PP +pynifti_pst computes the peristimulus signal timecourse for all voxels in a +volume at once. Stimulus onsets can be specified as volume numbers or times +(will be converted into volume numbers using a supplied repetition time). +Onsets can be specified directly on the command line, but can also be read +from (multiple) files. Such file are assumed to list one onset per line (as +the first value). Empty lines are ignored. This enables pynifti_pst to use +e.g. FSL's custom EV files. If several files are specified, the read onsets +are combined to a single onset vector. pynifti_pst writes a 4d timeseries +image as output. This image can e.g. be loaded into FSLView to look at each +voxels signal timecourse in a certain condition by simply clicking on it. +.SS "options:" +.TP +\fB\-\-version\fR +show program's version number and exit +.TP +\fB\-h\fR, \fB\-\-help\fR +show this help message and exit +.TP +\fB\-\-verbose\fR +print status messages +.TP +\fB\-n\fR NVOLS, \fB\-\-nvols\fR=\fINVOLS\fR +Set the length of the computed peristimulus signal +timecourse (in volumes). Default: 10 +.TP +\fB\-t\fR, \fB\-\-times\fR +If supplied, the read values are treated as onset +times and will be converted to volume numbers. For +each onset the volume that is closest in time will be +selected. Volumes are assumed to be recorded exactly +(and completely) after tr/2, e.g. if 'tr' is 2 secs +the first volume is recorded at exactly one second. +Please see the \fB\-\-tr\fR and \fB\-\-offset\fR options to learn how +to adjust the conversion. +.TP +\fB\-\-tr\fR=\fITR\fR +Repetition time of the 4d image (temporal difference +of two successive volumes). This can be used to +override the setting in the 4d image. The repetition +time is necessary to convert onset times into volume +numbers. If the '\-\-times' option is not set this value +has no effect. Please note that repetitions time and +the supplied onsets have to be in the same unit. +Please note, that if \fB\-\-times\fR is given the tr has to be +specified in the same unit as the read onset times. +.TP +\fB\-\-offset\fR=\fIOFFSET\fR +Constant offset applied to the onsets times when +converting them to volume numbers. Without setting '\-\- +times' this option has no effect'. +.TP +\fB\-p\fR, \fB\-\-percsigchg\fR +Convert the computed timecourse to percent signal +change relative to the first (onset) volume. This +might not be meaningful when \fB\-\-operation\fR is set to +something different than 'mean'. Please note, that the +shape of the computes timeseries heavily depends on +the first average volume. It might be more meaningful +to use a real baseline condition as origin. However, +this is not supported yet. Default: False +.TP +\fB\-\-printvoxel\fR=\fIPRINTVOXEL\fR +Print the peristimulus timeseries of a single voxel +for all onsets separately. This will print a matrix +(onsets x time), where the number of columns is +identical to the value of \fB\-\-nvols\fR and the number of +rows corresponds to the number of specified onsets. +(e.g. 'z,y,x') +.TP +\fB\-\-operation\fR=\fIOPERATION\fR +Choose the math operation that is performed to compute +the peristimulus timeseries. By default this is the +mean across all stimulations ('mean'). Other +possibilities are the standard deviation ('std') and +standard error ('sde'). Added: trunk/scipy/io/nifti/nifti/__init__.py =================================================================== --- trunk/scipy/io/nifti/nifti/__init__.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/nifti/__init__.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,18 @@ +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +# +# Import helper for PyNifti +# +# Copyright (C) 2006-2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### + +from niftiimage import * Added: trunk/scipy/io/nifti/nifti/nifticlib.i =================================================================== --- trunk/scipy/io/nifti/nifti/nifticlib.i 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/nifti/nifticlib.i 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,227 @@ +/*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** + * + * SWIG interface to wrap the low-level NIfTI IO libs for Python + * + * Copyright (C) 2006-2007 by + * Michael Hanke + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the MIT License. + * + * This package is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING + * file that comes with this package for more details. + * + *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** + */ + +%define DOCSTRING +" +This module provide the all functions and datatypes implemented +in the low-level C library libniftiio and the NIfTI-1 header. + +At the moment no additional documentation is provided here. Please see the +intensively documented source code of the nifti C libs to learn about the +capabilities of the library. +" +%enddef + +%module (package="nifti", docstring=DOCSTRING) nifticlib +%{ +#include +#include + +#include +#include + +/* low tech wrapper function to set values of a mat44 struct */ +static void set_mat44(mat44* m, + float a1, float a2, float a3, float a4, + float b1, float b2, float b3, float b4, + float c1, float c2, float c3, float c4, + float d1, float d2, float d3, float d4 ) +{ + m->m[0][0] = a1; m->m[0][1] = a2; m->m[0][2] = a3; m->m[0][3] = a4; + m->m[1][0] = b1; m->m[1][1] = b2; m->m[1][2] = b3; m->m[1][3] = b4; + m->m[2][0] = c1; m->m[2][1] = c2; m->m[2][2] = c3; m->m[2][3] = c4; + m->m[3][0] = d1; m->m[3][1] = d2; m->m[3][2] = d3; m->m[3][3] = d4; +} + +/* convert mat44 struct into a numpy float array */ +static PyObject* mat442array(mat44 _mat) +{ + int dims[2] = {4,4}; + + PyObject* array = 0; + array = PyArray_FromDims ( 2, dims, NPY_FLOAT ); + + /* mat44 subscription is [row][column] */ + PyArrayObject* a = (PyArrayObject*) array; + + float* data = (float *)a->data; + + int i,j; + + for (i = 0; i<4; i+=1) + { + for (j = 0; j<4; j+=1) + { + data[4*i+j] = _mat.m[i][j]; + } + } + + return PyArray_Return ( (PyArrayObject*) array ); +} + +static PyObject* wrapImageDataWithArray(nifti_image* _img) +{ + if (!_img) + { + PyErr_SetString(PyExc_RuntimeError, "Zero pointer passed instead of valid nifti_image struct."); + return(NULL); + } + + int array_type=0; + + /* translate nifti datatypes to numpy datatypes */ + switch(_img->datatype) + { + case NIFTI_TYPE_UINT8: + array_type = NPY_UBYTE; + break; + case NIFTI_TYPE_INT8: + array_type = NPY_BYTE; + break; + case NIFTI_TYPE_UINT16: + array_type = NPY_USHORT; + break; + case NIFTI_TYPE_INT16: + array_type = NPY_SHORT; + break; + case NIFTI_TYPE_UINT32: + array_type = NPY_UINT; + break; + case NIFTI_TYPE_INT32: + array_type = NPY_INT; + break; + case NIFTI_TYPE_UINT64: + case NIFTI_TYPE_INT64: + array_type = NPY_LONG; + break; + case NIFTI_TYPE_FLOAT32: + array_type = NPY_FLOAT; + break; + case NIFTI_TYPE_FLOAT64: + array_type = NPY_DOUBLE; + break; + case NIFTI_TYPE_COMPLEX128: + array_type = NPY_CFLOAT; + break; + case NIFTI_TYPE_COMPLEX256: + array_type = NPY_CDOUBLE; + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unsupported datatype"); + return(NULL); + } + + /* array object */ + PyObject* volarray = 0; + + /* Get the number of dimensions from the niftifile and + * reverse the order for the conversion to a numpy array. + * Doing so will make users access to the data more convenient: + * a 3d volume from a 4d file can be index by a single number. + */ + int ar_dim[7], ndims, k; + /* first item in dim array stores the number of dims */ + ndims = (int) _img->dim[0]; + /* reverse the order */ + for (k=0; kdim[ndims-k]; + } + + /* create numpy array */ + volarray = PyArray_FromDimsAndData ( ndims, ar_dim, array_type, ( char* ) _img->data ); + + return PyArray_Return ( (PyArrayObject*) volarray ); +} + +int allocateImageMemory(nifti_image* _nim) +{ + if (_nim == NULL) + { + fprintf(stderr, "NULL pointer passed to allocateImageMemory()"); + return(0); + } + + if (_nim->data != NULL) + { + fprintf(stderr, "There seems to be allocated memory already (valid nim->data pointer found)."); + return(0); + } + + /* allocate memory */ + _nim->data = (void*) calloc(1,nifti_get_volsize(_nim)); + + if (_nim->data == NULL) + { + fprintf(stderr, "Failed to allocate %d bytes for image data\n", (int)nifti_get_volsize(_nim)); + return(0); + } + + return(1); +} + +%} + + +%init +%{ + import_array(); +%} + +%include "typemaps.i" +/* Need to put before nifti1_io.h to overwrite function prototype with this + * typemap. */ +void nifti_mat44_to_quatern( mat44 R , + float *OUTPUT, float *OUTPUT, float *OUTPUT, + float *OUTPUT, float *OUTPUT, float *OUTPUT, + float *OUTPUT, float *OUTPUT, float *OUTPUT, float *OUTPUT ); + +void nifti_mat44_to_orientation( mat44 R , int *OUTPUT, int *OUTPUT, int *OUTPUT ); + + +%include znzlib.h +%include nifti1.h +%include nifti1_io.h + +static PyObject * wrapImageDataWithArray(nifti_image* _img); +int allocateImageMemory(nifti_image* _nim); + +static PyObject* mat442array(mat44 _mat); +static void set_mat44(mat44* m, + float a1, float a2, float a3, float a4, + float b1, float b2, float b3, float b4, + float c1, float c2, float c3, float c4, + float d1, float d2, float d3, float d4 ); + + +%include "cpointer.i" +%pointer_functions(short, shortp); +%pointer_functions(int, intp); +%pointer_functions(unsigned int, uintp); +%pointer_functions(float, floatp); +%pointer_functions(double, doublep); +%pointer_functions(char, charp); + +%include "carrays.i" +%array_class(short, shortArray); +%array_class(int, intArray); +%array_class(unsigned int, uintArray); +%array_class(float, floatArray); +%array_class(double, doubleArray); + + Added: trunk/scipy/io/nifti/nifti/nifticlib.py =================================================================== Added: trunk/scipy/io/nifti/nifti/niftiimage.py =================================================================== --- trunk/scipy/io/nifti/nifti/niftiimage.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/nifti/niftiimage.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,1244 @@ +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# Python interface to the NIfTI file format +# +# Copyright (C) 2006-2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + +# the swig wrapper if the NIfTI C library +import nifticlib +import os +import numpy + + +class NiftiImage(object): + """Wrapper class for convenient access to NIfTI data. + + The class can either load an image from file or convert a NumPy + array into a NIfTI file structure. Either way is automatically determined + by the type of the 'source' argument (string == filename, array == Numpy). + + One can optionally specify whether the image data should be loaded into + memory when opening NIfTI data from files ('load'). When converting a NumPy + array one can optionally specify a dictionary with NIfTI header data as + available via the 'header' attribute. + """ + + filetypes = [ 'ANALYZE', 'NIFTI', 'NIFTI_PAIR', 'ANALYZE_GZ', 'NIFTI_GZ', + 'NIFTI_PAIR_GZ' ] + + # map NumPy datatypes to NIfTI datatypes + numpy2nifti_dtype_map = { numpy.uint8: nifticlib.NIFTI_TYPE_UINT8, + numpy.int8 : nifticlib.NIFTI_TYPE_INT8, + numpy.uint16: nifticlib.NIFTI_TYPE_UINT16, + numpy.int16 : nifticlib.NIFTI_TYPE_INT16, + numpy.uint32: nifticlib.NIFTI_TYPE_UINT32, + numpy.int32 : nifticlib.NIFTI_TYPE_INT32, + numpy.uint64: nifticlib.NIFTI_TYPE_UINT64, + numpy.int64 : nifticlib.NIFTI_TYPE_INT64, + numpy.float32: nifticlib.NIFTI_TYPE_FLOAT32, + numpy.float64: nifticlib.NIFTI_TYPE_FLOAT64, + numpy.complex128: nifticlib.NIFTI_TYPE_COMPLEX128 + } + + + @staticmethod + def numpydtype2niftidtype(array): + """ Return the NIfTI datatype id for a corrsponding numpy array + datatype. + """ + # get the real datatype from numpy type dictionary + dtype = numpy.typeDict[str(array.dtype)] + + if not NiftiImage.numpy2nifti_dtype_map.has_key(dtype): + raise ValueError, "Unsupported datatype '%s'" % str(array.dtype) + + return NiftiImage.numpy2nifti_dtype_map[dtype] + + + @staticmethod + def splitFilename(filename): + """ Split a NIfTI filename and returns a tuple of basename and + extension. If no valid NIfTI filename extension is found, the whole + string is returned as basename and the extension string will be empty. + """ + + parts = filename.split('.') + + if parts[-1] == 'gz': + if not parts[-2] in [ 'nii', 'hdr', 'img' ]: + return filename, '' + else: + return '.'.join(parts[:-2]), '.'.join(parts[-2:]) + else: + if not parts[-1] in [ 'nii', 'hdr', 'img' ]: + return filename, '' + else: + return '.'.join(parts[:-1]), parts[-1] + + + @staticmethod + def nhdr2dict(nhdr): + """ Convert a NIfTI header struct into a python dictionary. + + While most elements of the header struct will be translated + 1:1 some (e.g. sform matrix) are converted into more convenient + datatypes (i.e. 4x4 matrix instead of 16 separate values). + """ + h = {} + + # the following header elements are converted in a simple loop + # as they do not need special handling + auto_convert = [ 'session_error', 'extents', 'sizeof_hdr', + 'slice_duration', 'slice_start', 'xyzt_units', + 'cal_max', 'intent_p1', 'intent_p2', 'intent_p3', + 'intent_code', 'sform_code', 'cal_min', 'scl_slope', + 'slice_code', 'bitpix', 'descrip', 'glmin', 'dim_info', + 'glmax', 'data_type', 'aux_file', 'intent_name', + 'vox_offset', 'db_name', 'scl_inter', 'magic', + 'datatype', 'regular', 'slice_end', 'qform_code', + 'toffset' ] + + + # now just dump all attributes into a dict + for attr in auto_convert: + h[attr] = eval('nhdr.' + attr) + + # handle a few special cases + # handle 'pixdim': carray -> list + pixdim = nifticlib.floatArray_frompointer(nhdr.pixdim) + h['pixdim'] = [ pixdim[i] for i in range(8) ] + + # handle dim: carray -> list + dim = nifticlib.shortArray_frompointer(nhdr.dim) + h['dim'] = [ dim[i] for i in range(8) ] + + # handle sform: carrays -> (4x4) numpy array + srow_x = nifticlib.floatArray_frompointer( nhdr.srow_x ) + srow_y = nifticlib.floatArray_frompointer( nhdr.srow_y ) + srow_z = nifticlib.floatArray_frompointer( nhdr.srow_z ) + + h['sform'] = numpy.array( [ [ srow_x[i] for i in range(4) ], + [ srow_y[i] for i in range(4) ], + [ srow_z[i] for i in range(4) ], + [ 0.0, 0.0, 0.0, 1.0 ] ] ) + + # handle qform stuff: 3 numbers -> list + h['quatern'] = [ nhdr.quatern_b, nhdr.quatern_c, nhdr.quatern_d ] + h['qoffset'] = [ nhdr.qoffset_x, nhdr.qoffset_y, nhdr.qoffset_z ] + + return h + + + @staticmethod + def updateNiftiHeaderFromDict(nhdr, hdrdict): + """ Update a NIfTI header struct with data from a dictionary. + + The supplied dictionary might contain additonal data elements + that do not match any nifti header element. These are silently ignored. + + Several checks are performed to ensure validity of the resulting + nifti header struct. If any check fails a ValueError exception will be + thrown. However, some tests are still missing. + """ + # this function is still incomplete. add more checks + + if hdrdict.has_key('data_type'): + if len(hdrdict['data_type']) > 9: + raise ValueError, \ + "Nifti header property 'data_type' must not be longer " \ + + "than 9 characters." + nhdr.data_type = hdrdict['data_type'] + if hdrdict.has_key('db_name'): + if len(hdrdict['db_name']) > 79: + raise ValueError, "Nifti header property 'db_name' must " \ + + "not be longer than 17 characters." + nhdr.db_name = hdrdict['db_name'] + + if hdrdict.has_key('extents'): + nhdr.extents = hdrdict['extents'] + if hdrdict.has_key('session_error'): + nhdr.session_error = hdrdict['session_error'] + + if hdrdict.has_key('regular'): + if len(hdrdict['regular']) > 1: + raise ValueError, \ + "Nifti header property 'regular' has to be a single " \ + + "character." + nhdr.regular = hdrdict['regular'] + if hdrdict.has_key('dim_info'): + if len(hdrdict['dim_info']) > 1: + raise ValueError, \ + "Nifti header property 'dim_info' has to be a " \ + + "single character." + nhdr.dim_info = hdrdict['dim_info'] + + if hdrdict.has_key('dim'): + dim = nifticlib.shortArray_frompointer(nhdr.dim) + for i in range(8): dim[i] = hdrdict['dim'][i] + if hdrdict.has_key('intent_p1'): + nhdr.intent_p1 = hdrdict['intent_p1'] + if hdrdict.has_key('intent_p2'): + nhdr.intent_p2 = hdrdict['intent_p2'] + if hdrdict.has_key('intent_p3'): + nhdr.intent_p3 = hdrdict['intent_p3'] + if hdrdict.has_key('intent_code'): + nhdr.intent_code = hdrdict['intent_code'] + if hdrdict.has_key('datatype'): + nhdr.datatype = hdrdict['datatype'] + if hdrdict.has_key('bitpix'): + nhdr.bitpix = hdrdict['bitpix'] + if hdrdict.has_key('slice_start'): + nhdr.slice_start = hdrdict['slice_start'] + if hdrdict.has_key('pixdim'): + pixdim = nifticlib.floatArray_frompointer(nhdr.pixdim) + for i in range(8): pixdim[i] = hdrdict['pixdim'][i] + if hdrdict.has_key('vox_offset'): + nhdr.vox_offset = hdrdict['vox_offset'] + if hdrdict.has_key('scl_slope'): + nhdr.scl_slope = hdrdict['scl_slope'] + if hdrdict.has_key('scl_inter'): + nhdr.scl_inter = hdrdict['scl_inter'] + if hdrdict.has_key('slice_end'): + nhdr.slice_end = hdrdict['slice_end'] + if hdrdict.has_key('slice_code'): + nhdr.slice_code = hdrdict['slice_code'] + if hdrdict.has_key('xyzt_units'): + nhdr.xyzt_units = hdrdict['xyzt_units'] + if hdrdict.has_key('cal_max'): + nhdr.cal_max = hdrdict['cal_max'] + if hdrdict.has_key('cal_min'): + nhdr.cal_min = hdrdict['cal_min'] + if hdrdict.has_key('slice_duration'): + nhdr.slice_duration = hdrdict['slice_duration'] + if hdrdict.has_key('toffset'): + nhdr.toffset = hdrdict['toffset'] + if hdrdict.has_key('glmax'): + nhdr.glmax = hdrdict['glmax'] + if hdrdict.has_key('glmin'): + nhdr.glmin = hdrdict['glmin'] + + if hdrdict.has_key('descrip'): + if len(hdrdict['descrip']) > 79: + raise ValueError, \ + "Nifti header property 'descrip' must not be longer " \ + + "than 79 characters." + nhdr.descrip = hdrdict['descrip'] + if hdrdict.has_key('aux_file'): + if len(hdrdict['aux_file']) > 23: + raise ValueError, \ + "Nifti header property 'aux_file' must not be longer " \ + + "than 23 characters." + nhdr.aux_file = hdrdict['aux_file'] + + if hdrdict.has_key('qform_code'): + nhdr.qform_code = hdrdict['qform_code'] + + if hdrdict.has_key('sform_code'): + nhdr.sform_code = hdrdict['sform_code'] + + if hdrdict.has_key('quatern'): + if not len(hdrdict['quatern']) == 3: + raise ValueError, \ + "Nifti header property 'quatern' must be float 3-tuple." + + nhdr.quatern_b = hdrdict['quatern'][0] + nhdr.quatern_c = hdrdict['quatern'][1] + nhdr.quatern_d = hdrdict['quatern'][2] + + if hdrdict.has_key('qoffset'): + if not len(hdrdict['qoffset']) == 3: + raise ValueError, \ + "Nifti header property 'qoffset' must be float 3-tuple." + + nhdr.qoffset_x = hdrdict['qoffset'][0] + nhdr.qoffset_y = hdrdict['qoffset'][1] + nhdr.qoffset_z = hdrdict['qoffset'][2] + + if hdrdict.has_key('sform'): + if not hdrdict['sform'].shape == (4,4): + raise ValueError, \ + "Nifti header property 'sform' must be 4x4 matrix." + + srow_x = nifticlib.floatArray_frompointer(nhdr.srow_x) + for i in range(4): srow_x[i] = hdrdict['sform'][0][i] + srow_y = nifticlib.floatArray_frompointer(nhdr.srow_y) + for i in range(4): srow_y[i] = hdrdict['sform'][1][i] + srow_z = nifticlib.floatArray_frompointer(nhdr.srow_z) + for i in range(4): srow_z[i] = hdrdict['sform'][2][i] + + if hdrdict.has_key('intent_name'): + if len(hdrdict['intent_name']) > 15: + raise ValueError, \ + "Nifti header property 'intent_name' must not be " \ + + "longer than 15 characters." + nhdr.intent_name = hdrdict['intent_name'] + + if hdrdict.has_key('magic'): + if hdrdict['magic'] != 'ni1' and hdrdict['magic'] != 'n+1': + raise ValueError, \ + "Nifti header property 'magic' must be 'ni1' or 'n+1'." + nhdr.magic = hdrdict['magic'] + + + def __init__(self, source, header = {}, load=False ): + """ Create a Niftifile object. + + This method decides whether to load a nifti image from file or create + one from array data, depending on the datatype of 'source'. If source + is a string, it is assumed to be a filename and an attempt will be made + to open the corresponding NIfTI file. If 'load' is set to True the image + data will be loaded into memory. + + If 'source' is a numpy array the array data will be used for the to be + created nifti image and a matching nifti header is generated. Additonal + header data might be supplied in a dictionary. However, dimensionality + and datatype are determined from the numpy array and not taken from + a header dictionary. + + If an object of a different type is supplied as 'source' a ValueError + exception will be thrown. + """ + + self.__nimg = None + + if type( source ) == numpy.ndarray: + self.__newFromArray( source, header ) + elif type ( source ) == str: + self.__newFromFile( source, load ) + else: + raise ValueError, \ + "Unsupported source type. Only NumPy arrays and filename " \ + + "string are supported." + + + def __del__(self): + """ Do all necessary cleanups by calling __close(). + """ + self.__close() + + + def __close(self): + """Close the file and free all unnecessary memory. + """ + if self.__nimg: + nifticlib.nifti_image_free(self.__nimg) + self.__nimg = None + + + def __newFromArray(self, data, hdr = {}): + """ Create a nifti image struct from a numpy array and optional header + data. + """ + + # check array + if len(data.shape) > 7: + raise ValueError, \ + "NIfTI does not support data with more than 7 dimensions." + + # create template nifti header struct + niptr = nifticlib.nifti_simple_init_nim() + nhdr = nifticlib.nifti_convert_nim2nhdr(niptr) + + # intermediate cleanup + nifticlib.nifti_image_free(niptr) + + # convert virgin nifti header to dict to merge properties + # with supplied information and array properties + hdic = NiftiImage.nhdr2dict(nhdr) + + # copy data from supplied header dict + for k, v in hdr.iteritems(): + hdic[k] = v + + # finally set header data that is determined by the data array + # convert numpy to nifti datatype + hdic['datatype'] = self.numpydtype2niftidtype(data) + + # make sure there are no zeros in the dim vector + # especially not in #4 as FSLView doesn't like that + hdic['dim'] = [ 1 for i in hdic['dim'] ] + + # set number of dims + hdic['dim'][0] = len(data.shape) + + # set size of each dim (and reverse the order to match nifti format + # requirements) + for i, s in enumerate(data.shape): + hdic['dim'][len(data.shape)-i] = s + + # set magic field to mark as nifti file + hdic['magic'] = 'n+1' + + # update nifti header with information from dict + NiftiImage.updateNiftiHeaderFromDict(nhdr, hdic) + + # make clean table + self.__close() + + # convert nifti header to nifti image struct + self.__nimg = nifticlib.nifti_convert_nhdr2nim(nhdr, 'pynifti_none') + + if not self.__nimg: + raise RuntimeError, "Could not create nifti image structure." + + # kill filename for nifti images from arrays + self.__nimg.fname = '' + self.__nimg.iname = '' + + # allocate memory for image data + if not nifticlib.allocateImageMemory(self.__nimg): + raise RuntimeError, "Could not allocate memory for image data." + + # assign data + self.data[:] = data[:] + + + def __newFromFile(self, filename, load=False): + """Open a NIfTI file. + + If there is already an open file it is closed first. If 'load' is True + the image data is loaded into memory. + """ + self.__close() + self.__nimg = nifticlib.nifti_image_read( filename, int(load) ) + + if not self.__nimg: + raise RuntimeError, "Error while opening nifti header." + + if load: + self.load() + + + def save(self, filename=None, filetype = 'NIFTI'): + """Save the image. + + If the image was created using array data (not loaded from a file) one + has to specify a filename. + + Setting the filename also determines the filetype (NIfTI/ANALYZE). + Please see the documentation of the setFilename() method for some + details on the 'filename' and 'filetype' argument. + + Calling save() without a specified filename on a NiftiImage loaded + from a file, will overwrite the original file. + + If not yet done already, the image data will be loaded into memory + before saving the file. + + Warning: There will be no exception if writing fails for any reason, + as the underlying function nifti_write_hdr_img() from libniftiio does + not provide any feedback. Suggestions for improvements are appreciated. + """ + + # If image data is not yet loaded, do it now. + # It is important to do it already here, because nifti_image_load + # depends on the correct filename set in the nifti_image struct + # and this will be modified in this function! + if not self.__haveImageData(): + self.load() + + # set a default description if there is none + if not self.description: + self.description = 'Created with PyNIfTI' + + # update header information + self.updateCalMinMax() + + # saving for the first time? + if not self.filename or filename: + if not filename: + raise ValueError, \ + "When saving an image for the first time a filename " \ + + "has to be specified." + + self.setFilename(filename, filetype) + + # now save it + nifticlib.nifti_image_write_hdr_img(self.__nimg, 1, 'wb') + # yoh comment: unfortunately return value of nifti_image_write_hdr_img + # can't be used to track the successful completion of save + # raise IOError, 'An error occured while attempting to save the image + # file.' + + + def __haveImageData(self): + """Returns true if the image data was loaded into memory. + or False if not. + + See: load(), unload() + """ + self.__ensureNiftiImage() + + if self.__nimg.data: + return True + else: + return False + + + def load(self): + """Load the image data into memory. + + It is save to call this method several times. + """ + self.__ensureNiftiImage() + + if nifticlib.nifti_image_load( self.__nimg ) < 0: + raise RuntimeError, "Unable to load image data." + + + def unload(self): + """Unload image data and free allocated memory. + """ + # if no filename is se, the data will be lost and cannot be recovered + if not self.filename: + raise RuntimeError, "No filename is set, unloading the data would " \ + + "loose it completely without a chance of recovery. " + self.__ensureNiftiImage() + + nifticlib.nifti_image_unload(self.__nimg) + + + def getDataArray(self): + """ Calls asarray(False) to return the NIfTI image data wrapped into + a NumPy array. + + Attention: The array shares the data with the NiftiImage object. Any + resize operation or datatype conversion will most likely result in a + fatal error. If you need to perform such things, get a copy + of the image data by using asarray(copy=True). + + The 'data' property is an alternative way to access this function. + """ + return self.asarray(False) + + + def asarray(self, copy = True): + """Convert the image data into a multidimensional array. + + Attention: If copy == False (the default) the array only wraps + the image data. Any modification done to the array is also done + to the image data. + + If copy is true the array contains a copy of the image data. + + Changing the shape, size or data of a wrapping array is not supported + and will most likely result in a fatal error. If you want to data + anything else to the data but reading or simple value assignment + use a copy of the data by setting the copy flag. Later you can convert + the modified data array into a NIfTi file again. + """ + self.__ensureNiftiImage() + + if not self.__haveImageData(): + self.load() + + a = nifticlib.wrapImageDataWithArray(self.__nimg) + + if copy: + return a.copy() + else: + return a + + + def getScaledData(self): + """ Returns a copy of the data array scaled by multiplying with the + slope and adding the intercept that is stored in the NIfTI header. + """ + data = self.asarray(copy = True) + + return data * self.slope + self.intercept + + + def __ensureNiftiImage(self): + """Check whether a NIfTI image is present. + + Returns True if there is a nifti image file structure or False + otherwise. One can create a file structure by calling open(). + """ + if not self.__nimg: + raise RuntimeError, "There is no NIfTI image file structure." + + + def updateCalMinMax(self): + """ Update the image data maximum and minimum value in the + nifti header. + """ + self.__nimg.cal_max = float(self.data.max()) + self.__nimg.cal_min = float(self.data.min()) + + + def getVoxDims(self): + """ Returns a 3-tuple a voxel dimensions/size in (x,y,z). + + The 'voxdim' property is an alternative way to access this function. + """ + return ( self.__nimg.dx, self.__nimg.dy, self.__nimg.dz ) + + + def setVoxDims(self, value): + """ Set voxel dimensions/size. + + This method takes a 3-tuple of floats as argument. The qform matrix + and its inverse will be recalculated automatically. + + Besides reading it is also possible to set the voxel dimensions by + assigning to the 'voxdim' property. + """ + if len(value) != 3: + raise ValueError, 'Requires 3-tuple.' + + self.__nimg.dx = float(value[0]) + self.__nimg.dy = float(value[1]) + self.__nimg.dz = float(value[2]) + + self.updateQFormFromQuaternion() + + + def setPixDims(self, value): + """ Set the pixel dimensions. + + The methods takes a sequence of up to 7 values (max. number of + dimensions supported by the NIfTI format. + + The supplied sequence can be shorter than seven elements. In this case + only present values are assigned starting with the first dimension + (spatial: x). Calling setPixDims() with a length-3 sequence equals + calling setVoxDims(). + """ + if len(value) > 7: + raise ValueError, \ + 'The Nifti format does not support more than 7 dimensions.' + + pixdim = nifticlib.floatArray_frompointer( self.__nimg.pixdim ) + + for i in value: + pixdim[i+1] = float(i) + + + def getPixDims(self): + """ Returns the pixel dimensions on all 7 dimensions. + + The function is similar to getVoxDims(), but instead of the 3d spatial + dimensions of a voxel it returns the dimensions of an image pixel on + all 7 dimensions supported by the NIfTI dataformat. + """ + return tuple( + [ nifticlib.floatArray_frompointer(self.__nimg.pixdim)[i] + for i in range(1,8) ] + ) + + + def getExtent(self): + """ Returns a tuple describing the shape (size in voxel/timepoints) + of the dataimage. + + The order of dimensions is (x,y,z,t,u,v,w). If the image has less + dimensions than 7 the return tuple will be shortened accordingly. + + Please note that the order of dimensions is different from the tuple + returned by calling NiftiImage.data.shape! + + See also getVolumeExtent() and getTimepoints(). + + The 'extent' property is an alternative way to access this function. + """ + # wrap dim array in nifti image struct + dims_array = nifticlib.intArray_frompointer(self.__nimg.dim) + dims = [ dims_array[i] for i in range(8) ] + + return tuple( dims[1:dims[0]+1] ) + + + def getVolumeExtent(self): + """ Returns the size/shape of the volume(s) in the image as a tuple. + + This method returns either a 3-tuple or 2-tuple or 1-tuple depending + on the available dimensions in the image. + + The order of dimensions in the tuple is (x [, y [, z ] ] ). + + The 'volextent' property is an alternative way to access this function. + """ + + # it is save to do this even if self.extent is shorter than 4 items + return self.extent[:3] + + + def getTimepoints(self): + """ Returns the number of timepoints in the image. + + In case of a 3d (or less dimension) image this method returns 1. + + The 'timepoints' property is an alternative way to access this + function. + """ + + if len(self.extent) < 4: + return 1 + else: + return self.extent[3] + + + def getRepetitionTime(self): + """ Returns the temporal distance between the volumes in a timeseries. + + The 'rtime' property is an alternative way to access this function. + """ + return self.__nimg.dt + + + def setRepetitionTime(self, value): + """ Set the repetition time of a nifti image (dt). + """ + self.__nimg.dt = float(value) + + + def getHeader(self): + """ Returns the header data of the nifti image in a dictionary. + + Note, that modifications done to this dictionary do not cause any + modifications in the NIfTI image. Please use the updateHeader() method + to apply changes to the image. + + The 'header' property is an alternative way to access this function. + But please note that the 'header' property cannot be used like this: + + nimg.header['something'] = 'new value' + + Instead one has to get the header dictionary, modify and later reassign + it: + + h = nimg.header + h['something'] = 'new value' + nimg.header = h + """ + h = {} + + # Convert nifti_image struct into nifti1 header struct. + # This get us all data that will actually make it into a + # NIfTI file. + nhdr = nifticlib.nifti_convert_nim2nhdr(self.__nimg) + + return NiftiImage.nhdr2dict(nhdr) + + + def updateHeader(self, hdrdict): + """ Update NIfTI header information. + + Updated header data is read from the supplied dictionary. One cannot + modify dimensionality and datatype of the image data. If such + information is present in the header dictionary it is removed before + the update. If resizing or datatype casting are required one has to + convert the image data into a separate array + ( NiftiImage.assarray(copy=True) ) and perform resize and data + manipulations on this array. When finished, the array can be converted + into a nifti file by calling the NiftiImage constructor with the + modified array as 'source' and the nifti header of the original + NiftiImage object as 'header'. + + It is save to call this method with and without loaded image data. + + The actual update is done by NiftiImage.updateNiftiHeaderFromDict(). + + Besides reading it is also possible to set the header data by assigning + to the 'header' property. Please see the documentation of the + getHeader() method for important information about the special usage + of the 'header' property. + """ + # rebuild nifti header from current image struct + nhdr = nifticlib.nifti_convert_nim2nhdr(self.__nimg) + + # remove settings from the hdrdict that are determined by + # the data set and must not be modified to preserve data integrity + if hdrdict.has_key('datatype'): + del hdrdict['datatype'] + if hdrdict.has_key('dim'): + del hdrdict['dim'] + + # update the nifti header + NiftiImage.updateNiftiHeaderFromDict(nhdr, hdrdict) + + # if no filename was set already (e.g. image from array) set a temp + # name now, as otherwise nifti_convert_nhdr2nim will fail + have_temp_filename = False + if not self.filename: + self.filename = 'pynifti_updateheader_temp_name' + have_temp_filename = True + + # recreate nifti image struct + new_nimg = nifticlib.nifti_convert_nhdr2nim(nhdr, self.filename) + if not new_nimg: + raise RuntimeError, \ + "Could not recreate NIfTI image struct from updated header." + + # replace old image struct by new one + # be careful with memory leak (still not checked whether successful) + + # rescue data ptr + new_nimg.data = self.__nimg.data + + # and remove it from old image struct + self.__nimg.data = None + + # to be able to call the cleanup function without lossing the data + self.__close() + + # assign the new image struct + self.__nimg = new_nimg + + # reset filename if temp name was set + if have_temp_filename: + self.filename = '' + + + def setSlope(self, value): + """ Set the slope attribute in the NIfTI header. + + Besides reading it is also possible to set the slope by assigning + to the 'slope' property. + """ + self.__nimg.scl_slope = float(value) + + + def setIntercept(self, value): + """ Set the intercept attribute in the NIfTI header. + + Besides reading it is also possible to set the intercept by assigning + to the 'intercept' property. + """ + self.__nimg.scl_inter = float(value) + + + def setDescription(self, value): + """ Set the description element in the NIfTI header. + + Descriptions must not be longer than 79 characters. + + Besides reading it is also possible to set the description by assigning + to the 'description' property. + """ + if len(value) > 79: + raise ValueError, \ + "The NIfTI format only supports descriptions shorter than " \ + + "80 chars." + + self.__nimg.descrip = value + + + def getSForm(self): + """ Returns the sform matrix. + + The 'sform' property is an alternative way to access this function. + + Please note, that the returned SForm matrix is not bound to the + NiftiImage object. Therefore it cannot be successfully modified + in-place. Modifications to the SForm matrix can only be done by setting + a new SForm matrix either by calling setSForm() or by assigning it to + the sform attribute. + """ + return nifticlib.mat442array(self.__nimg.sto_xyz) + + + def setSForm(self, m): + """ Sets the sform matrix. + The supplied value has to be a 4x4 matrix. The matrix elements will be + converted to floats. By definition the last row of the sform matrix has + to be (0,0,0,1). However, different values can be assigned, but will + not be stored when the niftifile is saved. + + The inverse sform matrix will be automatically recalculated. + + Besides reading it is also possible to set the sform matrix by + assigning to the 'sform' property. + """ + if m.shape != (4,4): + raise ValueError, "SForm matrix has to be of size 4x4." + + # make sure it is float + m = m.astype('float') + + nifticlib.set_mat44( self.__nimg.sto_xyz, + m[0,0], m[0,1], m[0,2], m[0,3], + m[1,0], m[1,1], m[1,2], m[1,3], + m[2,0], m[2,1], m[2,2], m[2,3], + m[3,0], m[3,1], m[3,2], m[3,3] ) + + # recalculate inverse + self.__nimg.sto_ijk = \ + nifticlib.nifti_mat44_inverse( self.__nimg.sto_xyz ) + + + def getInverseSForm(self): + """ Returns the inverse sform matrix. + + The 'sform_inv' property is an alternative way to access this function. + + Please note, that the inverse SForm matrix cannot be modified in-place. + One needs to set a new SForm matrix instead. The corresponding inverse + matrix is then re-calculated automatically. + """ + return nifticlib.mat442array(self.__nimg.sto_ijk) + + + def getQForm(self): + """ Returns the qform matrix. + + The 'qform' property is an alternative way to access this function. + + Please note, that the returned QForm matrix is not bound to the + NiftiImage object. Therefore it cannot be successfully modified + in-place. Modifications to the QForm matrix can only be done by setting + a new QForm matrix either by calling setSForm() or by assigning it to + the sform attribute. + """ + return nifticlib.mat442array(self.__nimg.qto_xyz) + + + def getInverseQForm(self): + """ Returns the inverse qform matrix. + + The 'qform_inv' property is an alternative way to access this function. + + Please note, that the inverse QForm matrix cannot be modified in-place. + One needs to set a new QForm matrix instead. The corresponding inverse + matrix is then re-calculated automatically. + """ + return nifticlib.mat442array(self.__nimg.qto_ijk) + + + def setQForm(self, m): + """ Sets the qform matrix. + The supplied value has to be a 4x4 matrix. The matrix will be converted + to float. + + The inverse qform matrix and the quaternion representation will be + automatically recalculated. + + Besides reading it is also possible to set the qform matrix by + assigning to the 'qform' property. + """ + if m.shape != (4,4): + raise ValueError, "QForm matrix has to be of size 4x4." + + # make sure it is float + m = m.astype('float') + + nifticlib.set_mat44( self.__nimg.qto_xyz, + m[0,0], m[0,1], m[0,2], m[0,3], + m[1,0], m[1,1], m[1,2], m[1,3], + m[2,0], m[2,1], m[2,2], m[2,3], + m[3,0], m[3,1], m[3,2], m[3,3] ) + + # recalculate inverse + self.__nimg.qto_ijk = \ + nifticlib.nifti_mat44_inverse( self.__nimg.qto_xyz ) + + # update quaternions + ( self.__nimg.quatern_b, self.__nimg.quatern_c, self.__nimg.quatern_d, + self.__nimg.qoffset_x, self.__nimg.qoffset_y, self.__nimg.qoffset_z, + self.__nimg.dx, self.__nimg.dy, self.__nimg.dz, + self.__nimg.qfac ) = \ + nifticlib.nifti_mat44_to_quatern( self.__nimg.qto_xyz ) + + + def updateQFormFromQuaternion(self): + """ Recalculates the qform matrix (and the inverse) from the quaternion + representation. + """ + # recalculate qform + self.__nimg.qto_xyz = nifticlib.nifti_quatern_to_mat44 ( + self.__nimg.quatern_b, self.__nimg.quatern_c, self.__nimg.quatern_d, + self.__nimg.qoffset_x, self.__nimg.qoffset_y, self.__nimg.qoffset_z, + self.__nimg.dx, self.__nimg.dy, self.__nimg.dz, + self.__nimg.qfac ) + + + # recalculate inverse + self.__nimg.qto_ijk = \ + nifticlib.nifti_mat44_inverse( self.__nimg.qto_xyz ) + + + def setQuaternion(self, value): + """ Set Quaternion from 3-tuple (qb, qc, qd). + + The qform matrix and its inverse are re-computed automatically. + + Besides reading it is also possible to set the quaternion by assigning + to the 'quatern' property. + """ + if len(value) != 3: + raise ValueError, 'Requires 3-tuple.' + + self.__nimg.quatern_b = float(value[0]) + self.__nimg.quatern_c = float(value[1]) + self.__nimg.quatern_d = float(value[2]) + + self.updateQFormFromQuaternion() + + + def getQuaternion(self): + """ Returns a 3-tuple containing (qb, qc, qd). + + The 'quatern' property is an alternative way to access this function. + """ + return( ( self.__nimg.quatern_b, + self.__nimg.quatern_c, + self.__nimg.quatern_d ) ) + + + def setQOffset(self, value): + """ Set QOffset from 3-tuple (qx, qy, qz). + + The qform matrix and its inverse are re-computed automatically. + + Besides reading it is also possible to set the qoffset by assigning + to the 'qoffset' property. + """ + if len(value) != 3: + raise ValueError, 'Requires 3-tuple.' + + self.__nimg.qoffset_x = float(value[0]) + self.__nimg.qoffset_y = float(value[1]) + self.__nimg.qoffset_z = float(value[2]) + + self.updateQFormFromQuaternion() + + + def getQOffset(self): + """ Returns a 3-tuple containing (qx, qy, qz). + + The 'qoffset' property is an alternative way to access this function. + """ + return( ( self.__nimg.qoffset_x, + self.__nimg.qoffset_y, + self.__nimg.qoffset_z ) ) + + + def setQFac(self, value): + """ Set qfac. + + The qform matrix and its inverse are re-computed automatically. + + Besides reading it is also possible to set the qfac by assigning + to the 'qfac' property. + """ + self.__nimg.qfac = float(value) + self.updateQFormFromQuaternion() + + + def getQOrientation(self, as_string = False): + """ Returns to orientation of the i,j and k axis as stored in the + qform matrix. + + By default NIfTI orientation codes are returned, but if 'as_string' is + set to true a string representation ala 'Left-to-right' is returned + instead. + """ + codes = nifticlib.nifti_mat44_to_orientation(self.__nimg.qto_xyz) + if as_string: + return [ nifticlib.nifti_orientation_string(i) for i in codes ] + else: + return codes + + + def getSOrientation(self, as_string = False): + """ Returns to orientation of the i,j and k axis as stored in the + sform matrix. + + By default NIfTI orientation codes are returned, but if 'as_string' is + set to true a string representation ala 'Left-to-right' is returned + instead. + """ + codes = nifticlib.nifti_mat44_to_orientation(self.__nimg.sto_xyz) + if as_string: + return [ nifticlib.nifti_orientation_string(i) for i in codes ] + else: + return codes + + + def getBoundingBox(self): + """ Get the bounding box of the image. + + This functions returns a tuple of (min, max) tuples. It contains as + many tuples as image dimensions. The order of dimensions is identical + to that in the data array. + + The 'bbox' property is an alternative way to access this function. + """ + nz = self.data.squeeze().nonzero() + + bbox = [] + + for dim in nz: + bbox.append( ( dim.min(), dim.max() ) ) + + return tuple(bbox) + + + def setFilename(self, filename, filetype = 'NIFTI'): + """ Set the filename for the NIfTI image. + + Setting the filename also determines the filetype. If the filename + ends with '.nii' the type will be set to NIfTI single file. A '.hdr' + extension can be used for NIfTI file pairs. If the desired filetype + is ANALYZE the extension should be '.img'. However, one can use the + '.hdr' extension and force the filetype to ANALYZE by setting the + filetype argument to ANALYZE. Setting filetype if the filename + extension is '.nii' has no effect, the file will always be in NIFTI + format. + + If the filename carries an additional '.gz' the resulting file(s) will + be compressed. + + Uncompressed NIfTI single files are the default filetype that will be + used if the filename has no valid extension. The '.nii' extension is + appended automatically. The 'filetype' argument can be used to force a + certain filetype when no extension can be used to determine it. + 'filetype' can be one of the nifticlibs filtetypes or any of 'NIFTI', + 'NIFTI_GZ', 'NIFTI_PAIR', 'NIFTI_PAIR_GZ', 'ANALYZE', 'ANALYZE_GZ'. + + Setting the filename will cause the image data to be loaded into memory + if not yet done already. This has to be done, because without the + filename of the original image file there would be no access to the + image data anymore. As a side-effect a simple operation like setting a + filename may take a significant amount of time (e.g. for a large 4d + dataset). + + By passing an empty string or none as filename one can reset the + filename and detach the NiftiImage object from any file on disk. + + Examples: + + Filename Output of save() + ---------------------------------- + exmpl.nii exmpl.nii (NIfTI) + exmpl.hdr exmpl.hdr, exmpl.img (NIfTI) + exmpl.img exmpl.hdr, exmpl.img (ANALYZE) + exmpl exmpl.nii (NIfTI) + exmpl.hdr.gz exmpl.hdr.gz, exmpl.img.gz (NIfTI) + + ! exmpl.gz exmpl.gz.nii (uncompressed NIfTI) + + Setting the filename is also possible by assigning to the 'filename' + property. + """ + # If image data is not yet loaded, do it now. + # It is important to do it already here, because nifti_image_load + # depends on the correct filename set in the nifti_image struct + # and this will be modified in this function! + if not self.__haveImageData(): + self.load() + + # if no filename is given simply reset it to nothing + if not filename: + self.__nimg.fname = '' + self.__nimg.iname = '' + return + + # separate basename and extension + base, ext = NiftiImage.splitFilename(filename) + + # if no extension default to nifti single files + if ext == '': + if filetype == 'NIFTI' \ + or filetype == nifticlib.NIFTI_FTYPE_NIFTI1_1: + ext = 'nii' + elif filetype == 'NIFTI_PAIR' \ + or filetype == nifticlib.NIFTI_FTYPE_NIFTI1_2: + ext = 'hdr' + elif filetype == 'ANALYZE' \ + or filetype == nifticlib.NIFTI_FTYPE_ANALYZE: + ext = 'img' + elif filetype == 'NIFTI_GZ': + ext = 'nii.gz' + elif filetype == 'NIFTI_PAIR_GZ': + ext = 'hdr.gz' + elif filetype == 'ANALYZE_GZ': + ext = 'img.gz' + else: + raise RuntimeError, "Unhandled filetype." + + # Determine the filetype and set header and image filename + # appropriately. + + # nifti single files are easy + if ext == 'nii.gz' or ext == 'nii': + self.__nimg.fname = base + '.' + ext + self.__nimg.iname = base + '.' + ext + self.__nimg.nifti_type = nifticlib.NIFTI_FTYPE_NIFTI1_1 + # uncompressed nifti file pairs + elif ext in [ 'hdr', 'img' ]: + self.__nimg.fname = base + '.hdr' + self.__nimg.iname = base + '.img' + if ext == 'hdr' and not filetype.startswith('ANALYZE'): + self.__nimg.nifti_type = nifticlib.NIFTI_FTYPE_NIFTI1_2 + else: + self.__nimg.nifti_type = nifticlib.NIFTI_FTYPE_ANALYZE + # compressed file pairs + elif ext in [ 'hdr.gz', 'img.gz' ]: + self.__nimg.fname = base + '.hdr.gz' + self.__nimg.iname = base + '.img.gz' + if ext == 'hdr.gz' and not filetype.startswith('ANALYZE'): + self.__nimg.nifti_type = nifticlib.NIFTI_FTYPE_NIFTI1_2 + else: + self.__nimg.nifti_type = nifticlib.NIFTI_FTYPE_ANALYZE + else: + raise RuntimeError, "Unhandled filetype." + + + def getFilename(self): + """ Returns the filename. + + To be consistent with setFilename() the image filename is returned + for ANALYZE images while the header filename is returned for NIfTI + files. + + The 'filename' property is an alternative way to access this function. + """ + if self.__nimg.nifti_type == nifticlib.NIFTI_FTYPE_ANALYZE: + return self.__nimg.iname + else: + return self.__nimg.fname + + # class properties + # read only + nvox = property(fget=lambda self: self.__nimg.nvox) + max = property(fget=lambda self: self.__nimg.cal_max) + min = property(fget=lambda self: self.__nimg.cal_min) + data = property(fget=getDataArray) + sform_inv = property(fget=getInverseSForm) + qform_inv = property(fget=getInverseQForm) + extent = property(fget=getExtent) + volextent = property(fget=getVolumeExtent) + timepoints = property(fget=getTimepoints) + bbox = property(fget=getBoundingBox) + + # read and write + filename = property(fget=getFilename, fset=setFilename) + slope = property(fget=lambda self: self.__nimg.scl_slope, + fset=setSlope) + intercept = property(fget=lambda self: self.__nimg.scl_inter, + fset=setIntercept) + voxdim = property(fget=getVoxDims, fset=setVoxDims) + pixdim = property(fget=getPixDims, fset=setPixDims) + description = property(fget=lambda self: self.__nimg.descrip, + fset=setDescription) + header = property(fget=getHeader, fset=updateHeader) + sform = property(fget=getSForm, fset=setSForm) + qform = property(fget=getQForm, fset=setQForm) + quatern = property(fget=getQuaternion, fset=setQuaternion) + qoffset = property(fget=getQOffset, fset=setQOffset) + qfac = property(fget=lambda self: self.__nimg.qfac, fset=setQFac) + rtime = property(fget=getRepetitionTime, fset=setRepetitionTime) + Added: trunk/scipy/io/nifti/nifti/utils.py =================================================================== --- trunk/scipy/io/nifti/nifti/utils.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/nifti/utils.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,129 @@ +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# Utility function for PyNifti +# +# Copyright (C) 2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + +import nifti +import numpy + +def time2vol( t, tr, lag=0.0, decimals=0 ): + """ Translates a time 't' into a volume number. By default function returns + the volume number that is closest in time. Volumes are assumed to be + recorded exactly (and completely) after tr/2, e.g. if 'tr' is 2 secs the + first volume is recorded at exactly one second. + + 't' might be a single value, a sequence or an array. + + The repetition 'tr' might be specified directly, but can also be a + NiftiImage object. In the latter case the value of 'tr' is determined from + the 'rtime' property of the NiftiImage object. + + 't' and 'tr' can be given in an arbitrary unit (but both have to be in the + same unit). + + The 'lag' argument can be used to shift the times by constant offset. + + Please note that numpy.round() is used to round to interger value (rounds + to even numbers). The 'decimals' argument will be passed to numpy.round(). + """ + # transform to numpy array for easy handling + tmp = numpy.array(t) + + # determine tr if NiftiImage object + if isinstance( tr, nifti.NiftiImage ): + tr = tr.rtime + + vol = numpy.round( ( tmp + lag + tr/2 ) / tr, decimals ) + + return vol + + +def applyFxToVolumes( ts, vols, fx, **kwargs ): + """ Apply a function on selected volumes of a timeseries. + + 'ts' is a 4d timeseries. It can be a NiftiImage or a numpy array. + In case of a numpy array one has to make sure that the time is on the + first axis. 'ts' can actually be of any dimensionality, but datasets aka + volumes are assumed to be along the first axis. + + 'vols' is either a sequence of sequences or a 2d array indicating which + volumes fx should be applied to. Each row defines a set of volumes. + + 'fx' is a callable function to get an array of the selected volumes as + argument. Additonal arguments may be specified as keyword arguments and + are passed to 'fx'. + + The output will be a 4d array with one computed volume per row in the 'vols' + array. + """ + # get data array from nifti image or assume data array is + # already present + if isinstance( ts, nifti.NiftiImage ): + data = ts.data + else: + data = ts + + out = [] + + for vol in vols: + out.append( fx( data[ numpy.array( vol ) ], **kwargs ) ) + + return numpy.array( out ) + + +def cropImage( nimg, bbox ): + """ Crop an image. + + 'bbox' has to be a sequency of (min,max) tuples (one for each image + dimension). + + The function returns the cropped image. The data is not shared with the + original image, but is copied. + """ + + # build crop command + cmd = 'nimg.data.squeeze()[' + cmd += ','.join( [ ':'.join( [ str(i) for i in dim ] ) for dim in bbox ] ) + cmd += ']' + + # crop the image data array + cropped = eval(cmd).copy() + + # return the cropped image with preserved header data + return nifti.NiftiImage(cropped, nimg.header) + + +def getPeristimulusTimeseries( ts, onsetvols, nvols = 10, fx = numpy.mean ): + """ Returns 4d array with peristimulus timeseries. + + Parameters: + ts - source 4d timeseries + onsetvols - sequence of onsetvolumes to be averaged over + nvols - length of the peristimulus timeseries in volumes + (starting from onsetvol) + fx - function to be applied to the list of corresponding + volumes. Typically this will be mean(), so it is default, + but it could also be var() or something different. The + supplied function is to be able to handle an 'axis=0' + argument similiar to NumPy's mean(), var(), ... + """ + selected = [ [ o + offset for o in onsetvols ] \ + for offset in range( nvols ) ] + + if fx == tuple: + return applyFxToVolumes( ts, selected, fx ) + else: + return applyFxToVolumes( ts, selected, fx, axis=0 ) + Added: trunk/scipy/io/nifti/setup.py =================================================================== --- trunk/scipy/io/nifti/setup.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/setup.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +# +# Python distutils setup for PyNifti +# +# Copyright (C) 2006-2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### + +from distutils.core import setup, Extension +import os +import numpy +from glob import glob + +nifti_wrapper_file = os.path.join('nifti', 'nifticlib.py') + +# create an empty file to workaround crappy swig wrapper installation +if not os.path.isfile(nifti_wrapper_file): + open(nifti_wrapper_file, 'w') + +# find numpy headers +numpy_headers = os.path.join(os.path.dirname(numpy.__file__),'core','include') + + +# Notes on the setup +# Version scheme is: +# 0.<4-digit-year><2-digit-month><2-digit-day>. + +setup(name = 'pynifti', + version = '0.20070930.1', + author = 'Michael Hanke', + author_email = 'michael.hanke at gmail.com', + license = 'MIT License', + url = 'http://apsy.gse.uni-magdeburg.de/hanke', + description = 'Python interface for the NIfTI IO libraries', + long_description = """ """, + packages = [ 'nifti' ], + scripts = glob( 'bin/*' ), + ext_modules = [ Extension( 'nifti._nifticlib', [ 'nifti/nifticlib.i' ], + include_dirs = [ '/usr/include/nifti', numpy_headers ], + libraries = [ 'niftiio' ], + swig_opts = [ '-I/usr/include/nifti', + '-I' + numpy_headers ] ) ] + ) + Added: trunk/scipy/io/nifti/tests/data/example4d.nii.gz =================================================================== (Binary files differ) Property changes on: trunk/scipy/io/nifti/tests/data/example4d.nii.gz ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Added: trunk/scipy/io/nifti/tests/test_fileio.py =================================================================== --- trunk/scipy/io/nifti/tests/test_fileio.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/tests/test_fileio.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,81 @@ +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +# +# Unit tests for PyNIfTI file io +# +# Copyright (C) 2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### + +import nifti +import unittest +import md5 +import tempfile +import shutil +import os +import numpy as np + + +def md5sum(filename): + """ Generate MD5 hash string. + """ + file = open( filename ) + sum = md5.new() + while True: + data = file.read() + if not data: + break + sum.update(data) + return sum.hexdigest() + + +class FileIOTests(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp('pynifti_test') + + def tearDown(self): + shutil.rmtree(self.workdir) + + def testIdempotentLoadSaveCycle(self): + """ check if file is unchanged by load/save cycle. + """ + md5_orig = md5sum('data/example4d.nii.gz') + nimg = nifti.NiftiImage('data/example4d.nii.gz') + nimg.save( os.path.join( self.workdir, 'iotest.nii.gz') ) + md5_io = md5sum( os.path.join( self.workdir, 'iotest.nii.gz') ) + + self.failUnlessEqual(md5_orig, md5_io) + + def testQFormSetting(self): + nimg = nifti.NiftiImage('data/example4d.nii.gz') + # 4x4 identity matrix + ident = np.identity(4) + self.failIf( (nimg.qform == ident).all() ) + + # assign new qform + nimg.qform = ident + self.failUnless( (nimg.qform == ident).all() ) + + # test save/load cycle + nimg.save( os.path.join( self.workdir, 'qformtest.nii.gz') ) + nimg2 = nifti.NiftiImage( os.path.join( self.workdir, + 'qformtest.nii.gz') ) + + self.failUnless( (nimg.qform == nimg2.qform).all() ) + + +def suite(): + return unittest.makeSuite(FileIOTests) + + +if __name__ == '__main__': + unittest.main() + Added: trunk/scipy/io/nifti/tests/test_main.py =================================================================== --- trunk/scipy/io/nifti/tests/test_main.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/tests/test_main.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,42 @@ +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +# +# Main unit test interface for PyNIfTI +# +# Copyright (C) 2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### + +import unittest + +# list all test modules (without .py extension) +tests = [ 'test_fileio', + 'test_utils', + ] + + +# import all test modules +for t in tests: + exec 'import ' + t + + +if __name__ == '__main__': + + # load all tests suites + suites = [ eval(t + '.suite()') for t in tests ] + + # and make global test suite + ts = unittest.TestSuite( suites ) + + # finally run it + unittest.TextTestRunner().run( ts ) + + Added: trunk/scipy/io/nifti/tests/test_utils.py =================================================================== --- trunk/scipy/io/nifti/tests/test_utils.py 2007-10-03 22:43:43 UTC (rev 3396) +++ trunk/scipy/io/nifti/tests/test_utils.py 2007-10-03 23:58:26 UTC (rev 3397) @@ -0,0 +1,37 @@ +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### +# +# Unit tests for PyNIfTI file io +# +# Copyright (C) 2007 by +# Michael Hanke +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the MIT License. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING +# file that comes with this package for more details. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### + +import nifti.utils +import numpy +import unittest + + +class UtilsTests(unittest.TestCase): + def testZScoring(self): + # dataset: mean=2, std=1 + data = numpy.array( (0,1,3,4,2,2,3,1,1,3,3,1,2,2,2,2) ) + self.failUnlessEqual( data.mean(), 2.0 ) + self.failUnlessEqual( data.std(), 1.0 ) + + +def suite(): + return unittest.makeSuite(UtilsTests) + + +if __name__ == '__main__': + unittest.main() + From scipy-svn at scipy.org Wed Oct 3 20:16:52 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 3 Oct 2007 19:16:52 -0500 (CDT) Subject: [Scipy-svn] r3399 - trunk/scipy/io/nifti Message-ID: <20071004001652.693D7C7C00A@new.scipy.org> Author: chris.burns Date: 2007-10-03 19:16:50 -0500 (Wed, 03 Oct 2007) New Revision: 3399 Added: trunk/scipy/io/nifti/README Removed: trunk/scipy/io/nifti/README.html Log: Replace README.html with a reST version. Added: trunk/scipy/io/nifti/README =================================================================== --- trunk/scipy/io/nifti/README 2007-10-04 00:12:15 UTC (rev 3398) +++ trunk/scipy/io/nifti/README 2007-10-04 00:16:50 UTC (rev 3399) @@ -0,0 +1,310 @@ +======================================================== +PyNIfTI - Python-style access to NIfTI and ANALYZE files +======================================================== + +.. Contents:: + +1. What is NIfTI and what do I need PyNIfTI for? +------------------------------------------------ + +NIfTI ++++++ + +NIfTI is a new Analyze-style data format, proposed by the NIfTI Data Format Working Group as a "short-term measure to facilitate inter-operation of functional MRI data analysis software packages". + +Meanwhile a number of toolkits are NIfTI-aware (e.g. FSL, AFNI, SPM, Freesurfer and Brainvoyager). Additionally, dicomnifti allows the direct conversion from DICOM images into the NIfTI format. + +With libniftiio there is a reference implementation of a C library to read, write and manipulate NIfTI images. The library source code is put into the public domain and a corresponding project is hosted at SourceForge. + +In addition to the C library, there is also an IO library written in Java and Matlab functions to make use of NIfTI files from within Matlab. + +Python +++++++ + +Unfortunately, it is not that trivial to read NIfTI images with Python. This is particularly sad, because there is a large number of easy-to-use, high-quality libraries for signal processing available for Python (e.g. SciPy). + +Moreover Python has bindings to almost any important language/program in the fields of maths, statistics and/or engineering. If you want to use R to calculate some stats in a Python script, simply use RPy and pass any data to R. If you don't care about R, but Matlab is your one and only friend, there are at least two different Python modules to control Matlab from within Python scripts. Python is the glue between all those helpers and the Python user is able to combine as many tools as necessary to solve a given problem -- the easiest way. + +PyNIfTI ++++++++ + +PyNIfTI aims to provide easy access to NIfTI images from within Python. It uses SWIG-generated wrappers for the NIfTI reference library and provides the NiftiImage class for Python-style access to the image data. + +While PyNIfTI is not yet complete, it already provides access to the most important features of the NIfTI-1 data format and libniftiio capabilities. The following features are currently implemented: + + * PyNIfTI can read and write any file format supported by libniftiio. This includes NIfTI (single and pairs) as well as ANALYZE files. + * PyNIfTI provides fast and convenient access to the image data via NumPy arrays. This should enable users to process image data with most (if not all) numerical routines available for Python. The NumPy array automatically uses a datatype corresponding to the NIfTI image data -- no unnecessary upcasting is performed. + * PyNIfTI provides full read and write access to the NIfTI header data. Header information can be exported to a Python dictionary and can also be updated by using information from a dictionary. + * Instead of accessing NIfTI data from files, PyNIfTI is able to create NIfTI images from NumPy arrays. The appropriate NIfTI header information is determined from the array properties. Additional header information can be optionally specified -- making it easy to clone NIfTI images if necessary, but with minor modifications. + * Most properties of NIfTI images are accessible via attributes and/or accessor functions of the NiftiImage. Inter-dependent properties are automatically updated if necessary (e.g. modifying the Q-Form matrix also updates the pixdim properties and quaternion representation). + * All properties are accessible via Python-style datatypes: A 4x4 matrix is an array not 16 individual numbers. + * PyNIfTI should be resonably fast. Image data will only be loaded into the memory if necessary. Simply opening a NIfTI file to access some header data is performed with virtually no delay independent of the size of the image. Unless image resizing or datatype conversion must be performed the image data can be shared by the NIfTI image and accessing NumPy arrays, and therefore memory won't be wasted memory with redundant copies of the image data. However, one should be careful to make a copy of the image data if you intend to resize and cast the image data (see the docstring of the NiftiImage.asarray() method). + +Scripts ++++++++ + +Some functions provided by PyNIfTI also might be useful outside the Python environment. Therefore I plan to add some command line scripts to the package. +Currently there is only one: pynifti_pst (pst: peristimulus timecourse). Using this script one can compute the signal timecourse for a certain condition for all voxels in a volume at once. This might be useful for exploring a dataset and accompanies similar tools like FSL's tsplot. + +The output of pynifti_pst can be loaded into FSLView to simultaneously look at statistics and signal timecourses. Please see the corresponding example below. + +Known issues aka bugs ++++++++++++++++++++++ + + * PyNIfTI currently ignores the origin field of ANALYZE files - it is neither read nor written. A possible workaround is to convert ANALYZE files into the NIfTI format using FSL's avwchfiletype. + +2. License +---------- + +PyNIfTI is written by Michael Hanke as free software (both beer and speech) and licensed under the GNU Lesser General Public License. + +3. Download +----------- + +As PyNIfTI is still pretty young, a number of significant improvements/modifications are very likely to happen in the near future. If you discover any bugs or you are missing some features, please be sure to check the SVN repository (read below) if your problem is already solved. +Source code + +Since June 2007 PyNIfTI is part of the niftilibs family. The PyNIfTI source code can be obtained from the Sourceforge project site. + +Source code is also available from the SVN repository of the Debian Experimental Psychology Project. To checkout the latest version use this command: + +svn co svn://svn.debian.org/svn/pkg-exppsy/pynifti/trunk pynifti-latest +Binary packages + +Binary packages for some Debian and (K)Ubuntu versions are available. Please visit this page to read about how you have to setup your system to retrieve the PyNIfTI package via your package manager and stay in sync with future releases. + +4. Installation +--------------- + +Binary packages ++++++++++++++++ + +If you have configured your system as described on this page all you have to do to install PyNIfTI is this:: + + + apt-get update + apt-get install python-nifti + +This should pull all necessary dependencies. + +Compile from source ++++++++++++++++++++ + +PyNIfTI needs a few things to build and run properly: + + * Python 2.3 or greater + * NumPy + * SWIG + * NIfTI C libraries + +Make sure that the compiled nifticlibs and the corresponding headers are available to your compiler. If they are located in a custom directory, you might have to specify --include-dirs and --library-dirs options to the build command below. + +Once you have downloaded the sources, extract the tarball and enter the root directory of the extracted sources. A simple:: + + python setup.py build_ext + +should build the SWIG wrappers. If this has been done successfully, all you need to do is install the modules by invoking:: + + sudo python setup.py install + +If sudo is not configured (or even installed) you might have to use su instead. + +Now fire up Python and try importing the module to see if everything is fine. It should look similar to this:: + + Python 2.4.4 (#2, Oct 20 2006, 00:23:25) + [GCC 4.1.2 20061015 (prerelease) (Debian 4.1.1-16.1)] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import nifti + >>> + +MacOS X and MacPython ++++++++++++++++++++++ + +When you are comiling PyNIfTI on MacOS X and want to use it with MacPython, please make sure that the NIfTI C libraries are compiled as fat binaries (compiled for both ppc and i386). Otherwise PyNIfTI extensions will not compile. + +One can achieve this by adding both architectures to the CFLAGS definition in the toplevel Makefile of the NIfTI C library source code. Like this:: + + CFLAGS = $(ANSI_FLAGS) -arch ppc -arch i386 + +Troubleshooting ++++++++++++++++ + +If you get an error when importing the nifti module in Python complaining about missing symbols your niftiio library contains references to some unresolved symbols. Try adding znzlib and zlib to the linker options the PyNIfTI setup.py, like this:: + + libraries = [ 'niftiio', 'znz', 'z' ], + +5. Things to know +----------------- + +When accessing NIfTI image data through NumPy arrays the order of the dimensions is reversed. If the x, y, z, t dimensions of a NIfTI image are 64, 64, 32, 456 (as for example reported by nifti_tool), the shape of the NumPy array (e.g. as returned by NiftiImage.asarray()) will be: 456, 32, 64, 64. + +This is done to be able to slice the data array much easier in the most common cases. For example, if you are interested in a certain volume of a timeseries it is much easier to write data[2] instead of data[:,:,:,2], right?. + +6. Examples +----------- + +The next sections contains some examples showing ways to use PyNIfTI to read and write imaging data from within Python to be able to process it with some random Python library. + +All examples assume that you have imported the PyNIfTI module by invoking:: + + from nifti import * + +a) Fileformat conversion +++++++++++++++++++++++++ + +Open the MNI standard space template that is shipped with FSL. No filename extension is necessary as libniftiio determines it automatically:: + + nim = NiftiImage('avg152T1_brain') + +The filename is available via the 'filename' attribute:: + + print nim.filename + +yields 'avg152T1_brain.img'. This indicates an ANALYZE image. If you want to save this image as a single gzipped NIfTI file simply do:: + + nim.save('mni.nii.gz') + +The filetype is determined from the filename. If you want to save to gzipped ANALYZE file pairs instead the following would be an alternative to calling the save() with a new filename:: + + nim.filename = 'mni_analyze.img.gz' + nim.save() + +Please see the docstring of the NiftiImage.setFilename() method to learn how the filetypes are determined from the filenames. + +b) NIfTI files from array data +++++++++++++++++++++++++++++++ + +The next code snipped demonstrates how to create a 4d NIfTI image containing gaussian noise. First we need to import the NumPy module:: + + import numpy + +Now generate the noise dataset. Let's generate noise for 100 volumes with 16 slices and a 32x32 inplane matrix:: + + noise = numpy.random.randn(100,16,32,32) + +Please notice the order in which the dimensions are specified: (t, z, y, x). + +The datatype of the array will most likely be float64 -- which can be verified by invoking noise.dtype. + +Converting this dataset into a NIfTI image is done by invoking the NiftiImage constructor with the noise dataset as argument:: + + nim = NiftiImage(noise) + +The relevant header information is extracted from the NumPy array. If you query the header information about the dimensionality of the image, it returns the desired values:: + + print nim.header['dim'] # yields: [4, 32, 32, 16, 100, 0, 0, 0] + +First value shows the number of dimensions in the datset: 4 (good, that's what we wanted). The following numbers are dataset size on the x, y, z, t, u, v, w axis (NIfTI files can handle up to 7 dimensions). Please notice, that the order of dimensions is now 'correct': We have 32x32 inplane resolution, 16 slices in z direction and 100 volumes. + +Also the datatype was set appropriately. The expression:: + + nim.header['datatype'] == nifticlib.NIFTI_TYPE_FLOAT64 + +will evaluate to True. + +To save the noise file to disk, just call the save() method:: + + nim.save('noise.nii.gz') + +c) Select ROIs +++++++++++++++ + +Suppose you want to have the first ten volumes of the noise dataset we have just created in a separate file. First open the file (can be skipped if it is still open):: + + nim = NiftiImage('noise.nii.gz') + +Now select the first ten volumes and store them to another file, while preserving as much header information as possible:: + + nim2 = NiftiImage(nim.data[:10], nim.header) + nim2.save('part.hdr.gz') + +The NiftiImage constructor takes a dictionary with header information as an optional argument. Settings that are not determined by the array (e.g. size, datatype) are copied from the dictionary and stored to the new NIfTI image. + +d) Linear detrending of timeseries +++++++++++++++++++++++++++++++++++ + +Let's load another 4d NIfTI file and perform a linear detrending, by fitting a straight line to the timeseries of each voxel and substract that fit from the data. Although this might sound complicated at first, thanks to the excellent SciPy module it is just a few lines of code:: + + nim = NiftiImage('timeseries.nii') + +Depending on the datatype of the input image the detrending process might change the datatype from integer to float. As operations that change the (binary) size of the NIfTI image are not supported, we need to make a copy of the data and later create a new NIfTI image:: + + data = nim.asarray() + +Now detrend the data along the time axis. Remember that the array has the time axis as its first dimension (in contrast to the NIfTI file where it is the 4th):: + + from scipy import signal + data_detrended = signal.detrend( data, axis=0 ) + +Finally, create a new NIfTI image using header information from the original source image:: + + nim_detrended = NiftiImage( data_detrended, nim.header) + +e) Make a quick plot of a voxels timeseries (Gnuplot module is required) +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Plotting is essential to get a 'feeling' for the data. The python interface to Gnuplot makes it really easy to plot something (e.g. when running Python interactively via IPython). Please note, that there are many other possibilities for plotting. Some examples are: using R via RPy or Matlab-style plotting via matplotlib. + +However, using Gnuplot is really easy. First import the Gnuplot module and create the interface object:: + + from Gnuplot import Gnuplot + gp = Gnuplot() + +We want the timeseries as a line plot and not just the datapoints, so let's talk with Gnuplot:: + + gp('set data style lines') + +Now load a 4d NIfTI image:: + + nim = NiftiImage('perfect_subject.nii.gz') + +and finally plot the timeseries of voxel (x=20, y=30, z=12):: + + gp.plot(nim.data[:,12,30,20]) + +A Gnuplot window showing the timeseries should popup now (screenshot). Please read the Gnuplot Manual to learn what it can do -- and it can do a lot more than just simple line plots (have a look at this page if you are interested). +f) Show a slice of a 3d volume (Matplotlib module is required) + +This example demonstrates howto use the Matlab-style plotting of Matplotlib to view a slice from a 3d volume. + +This time I assume that a 3d nifti file is already opened and available in the nim3d object. At first we need to load the necessary Python module:: + + from pylab import * + +If everything went fine, we can now view a slice (x,y):: + + imshow(nim3d.data[200], interpolation='nearest', cmap=cm.gray) + show() + +It is necessary to call the show() function one time after importing pylab to actually see the image when running Python interactively (screenshot). + +When you want to have a look at a yz-slice, NumPy array magic comes into play:: + + imshow(nim3d.data[::-1,:,100], interpolation='nearest', cmap=cm.gray) + +The ::-1 notation causes the z-axis to be flipped in the images. This makes a much nicer screenshot, because the used example volume has the z-axis originally oriented upsidedown. + +g) Compute and display peristimulus signal timecourse of multiple conditions with pynifti_pst and FSLView ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Sometimes one wants to look at the signal timecourse of some voxel after a certain stimulation onset. An easy way would be to have some fMRI data viewer that displays a statistical map and one could click on some activated voxel and the peristimulus signal timecourse of some condition in that voxel would be displayed. + +This can easily be done by using pynifti_pst and FSLView. + +pynifti_pst comes with a manpage that explains all options and arguments. Basically pynifti_pst need a 4d image (e.g. an fMRI timeseries; possibly preprocessed/filtered) and some stimulus onset information. This information can either be given directly on the command line or is read from files. Additionally one can specify onsets as volume numbers or as onset times. + +pynifti_pst understands the FSL custom EV file format so one can easily use those files as input. + +An example call could look like this: + +pynifti_pst --times --nvols 5 -p uf92.feat/filtered_func_data.nii.gz pst_cond_a.nii.gz uf92.feat/custom_timing_files/ev1.txt uf92.feat/custom_timing_files/ev2.txt + +This computes a peristimulus timeseries using the preprocessed fMRI from a FEAT output directory and two custom EV files that both together make up condition A. --times indicates that the EV files list onset times (not volume ids) and --nvols requests the mean peristimulus timecourse for 4 volumes after stimulus onset (5 including onset). -p recodes the peristimulus timeseries into percent signalchange, where the onset is always zero and any following value is the signal change with respect to the onset volume. + +This call produces a simple 4d NIfTI image that can be loaded into FSLView as any other timeseries. The following call can be used to display an FSL zmap from the above results path on top of some anatomy. Additionally the peristimulus timeseries of two conditions are loaded. This screenshot shows how it could look like. One of the nice features of FSLView is that its timeseries window can remember selected curves, which can be useful to compare signal timecourses from different voxels (blue and green line in the screenshot). + +fslview pst_cond_a.nii.gz pst_cond_b.nii.gz uf92_ana.nii.gz uf92.feat/stats/zstat1.nii.gz -b 3,5 + +History + +The full changelog is here. Deleted: trunk/scipy/io/nifti/README.html =================================================================== --- trunk/scipy/io/nifti/README.html 2007-10-04 00:12:15 UTC (rev 3398) +++ trunk/scipy/io/nifti/README.html 2007-10-04 00:16:50 UTC (rev 3399) @@ -1,391 +0,0 @@ - - - - - - PyNIfTI - Python bindings to NIfTI - - - - - - -

PyNIfTI - Python-style access to NIfTI and ANALYZE files

- - -

1. What is NIfTI and what do I need PyNIfTI for?

- -

NIfTI

-

NIfTI is a new Analyze-style data -format, proposed by the -NIfTI Data Format Working Group -as a "short-term measure to facilitate inter-operation of functional MRI data -analysis software packages".

-

Meanwhile a number of toolkits are NIfTI-aware (e.g. FSL, AFNI, SPM, -Freesurfer and a to a certain degree also Brainvoyager). -Additionally, dicomnifti -allows the direct conversion from DICOM images into the NIfTI format.

-

With libniftiio -there is a reference implementation of a C library to read, write and manipulate -NIfTI images. The library source code is put into the public domain and a -corresponding project is hosted at -SourceForge.

-

In addition to the C library, there is also an IO library written in Java and -Matlab functions to make use of NIfTI files from within Matlab.

- - -

Python

-

Unfortunately, it is not that trivial to read NIfTI images with Python. This is -particularly sad, because there is a large number of easy-to-use, high-quality -libraries for signal processing available for Python (e.g. SciPy).

-

Moreover Python has bindings to almost any important language/program -in the fields of maths, statistics and/or engineering. If you want to use -R to calculate -some stats in a Python script, simply use RPy -and pass any data to R. If you don't care about R, but Matlab is your one and -only friend, there are at least two different Python modules to control Matlab -from within Python scripts. Python is the glue between all those helpers and the Python user is -able to combine as many tools as necessary to solve a given problem --- the easiest way.

- - -

PyNIfTI

-

PyNIfTI aims to provide easy access to NIfTI images from within Python. It -uses SWIG-generated wrappers for the NIfTI -reference library and provides the NiftiImage class for -Python-style access to the image data.

-

While PyNIfTI is not yet complete (i.e. doesn't support everything the -C library can do), it already provides access to the most -important features of the NIfTI-1 data format and libniftiio -capabilities. The following features are currently implemented:

-
    - -
  • PyNIfTI can read and write any file format supported by libniftiio. This -includes NIfTI (single and pairs) as well as ANALYZE files.
    • -
  • PyNIfTI provides fast and convenient access to the image data via -NumPy arrays. This should enable users to -process image data with most (if not all) numerical routines available for -Python. The NumPy array automatically uses a datatype corresponding to the NIfTI image data -- no -unnecessary upcasting is performed.
    • -
  • PyNIfTI provides full read and write access to the NIfTI header data. Header information can -be exported to a Python dictionary and can also be updated by using information -from a dictionary.
    • -
  • Instead of accessing NIfTI data from files, PyNIfTI is able to create NIfTI -images from NumPy arrays. The appropriate NIfTI header information is determined -from the array properties. Additional header information can be optionally -specified -- making it easy to clone NIfTI images if necessary, but with minor -modifications.
    • -
  • Most properties of NIfTI images are accessible via attributes and/or accessor -functions of the NiftiImage. Inter-dependent properties are -automatically updated if necessary (e.g. modifying the Q-Form matrix also updates -the pixdim properties and quaternion representation).
    • -
  • All properties are accessible via Python-style datatypes: A 4x4 matrix is -an array not 16 individual numbers.
    • -
  • PyNIfTI should be resonably fast. Image data will only be loaded into the memory -if necessary. Simply opening a NIfTI file to access some header data is -performed with virtually no delay independent of the size of the image. Unless image -resizing or datatype conversion must be performed the image data can be -shared by the NIfTI image and accessing NumPy arrays, and therefore memory won't be wasted -memory with redundant copies of the image data. However, one should be careful to make a -copy of the image data if you intend to resize and cast the image data (see the -docstring of the NiftiImage.asarray() method).
    • -
- -

Scripts

-

Some functions provided by PyNIfTI also might be useful outside the Python -environment. Therefore I plan to add some command line scripts to the package.

-

Currently there is only one: pynifti_pst (pst: peristimulus -timecourse). Using this script one can compute the signal timecourse for a -certain condition for all voxels in a volume at once. This might be useful for -exploring a dataset and accompanies similar tools like FSL's tsplot.

-

The output of pynifti_pst can be loaded into FSLView to simultaneously -look at statistics and signal timecourses. Please see the corresponding example below.

- -

Known issues aka bugs

-
    -
  • PyNIfTI currently ignores the origin field of ANALYZE files - it is neither read - nor written. A possible workaround is to convert ANALYZE files into the NIfTI format - using FSL's avwchfiletype.
    • -
- - -

2. License

-

PyNIfTI is written by Michael Hanke -as free software (both beer and speech) and licensed under the -MIT License. -

- - -

3. Download

-

As PyNIfTI is still pretty young, a number of significant -improvements/modifications are very likely to happen in the near future. If you -discover any bugs or you are missing some features, please be sure to check the -SVN repository (read below) if your problem is already solved.

- -

-

Source code

-

Since June 2007 PyNIfTI is part of the -niftilibs family. The PyNIfTI -source code can be obtained from the -Sourceforge project site. -

- -

Binary packages

-

GNU/Linux

-

PyNIfTI is available in recent versions of the Debian (since lenny) and -Ubuntu (since gutsy in universe) distributions. The name of the binary package -is python-nifti in both cases.

- -

Binary packages for some additional Debian and (K)Ubuntu versions are also -available. Please visit -this page to read -about how you have to setup your system to retrieve the PyNIfTI package via -your package manager and stay in sync with future releases.

- -

Windows

-

A binary installer for a recent Python version is available from the -Sourceforge project site. - - -

Macintosh

-

Unfortunately, no binary packages are available. I have no access to such -a machine at the moment. But it is possible to build PyNIfTI from source on -Mac OS X (see below for more information).

- - - -

4. Installation

- -

Compile from source: General instructions

-

PyNIfTI needs a few things to build and run properly:

- -

Make sure that the compiled nifticlibs and the corresponding headers are -available to your compiler. If they are located in a custom directory, you -might have to specify --include-dirs and ---library-dirs options to the build command below.

-

Once you have downloaded the sources, extract the tarball and enter the root -directory of the extracted sources. A simple

-

python setup.py build_ext

-

should build the SWIG wrappers. If this has been done -successfully, all you need to do is install the modules by invoking

-

sudo python setup.py install

-

If sudo is not configured (or even installed) you might have to use -su instead.

-

Now fire up Python and try importing the module to see if everything is -fine. It should look similar to this:

-
-Python 2.4.4 (#2, Oct 20 2006, 00:23:25)
-[GCC 4.1.2 20061015 (prerelease) (Debian 4.1.1-16.1)] on linux2
-Type "help", "copyright", "credits" or "license" for more information.
->>> import nifti
->>> 
-
-

Windows

-

It should be pretty straightforward to compile PyNIfTI for win32. The most -convenient way seems to be using the -Dev-Cpp IDE -and the DevPak of the nifticlibs. Have a look into the toplevel Makefile of the -PyNIfTI source distribution for some hints.

- - -

MacOS X and MacPython

-

When you are comiling PyNIfTI on MacOS X and want to use it with MacPython, -please make sure that the NIfTI C libraries are compiled as fat binaries -(compiled for both ppc and i386). Otherwise -PyNIfTI extensions will not compile.

-

One can achieve this by adding both architectures to the CFLAGS -definition in the toplevel Makefile of the NIfTI C library source code. Like -this

-

CFLAGS = $(ANSI_FLAGS) -arch ppc -arch i386

- - -

Binary packages

-

GNU/Linux

-

If you have configured your system as described on -this page all you -have to do to install PyNIfTI is this:

-

apt-get update
apt-get install python-nifti

-

This should pull all necessary dependencies. If it doesn't, it's a bug that -should be reported.

-

Windows

-

As always: click Next as long as necessary and finally Finish. - -

Troubleshooting

-

If you get an error when importing the nifti module in Python -complaining about missing symbols your niftiio library contains references to -some unresolved symbols. Try adding znzlib and zlib to the -linker options the PyNIfTI setup.py, like this:

- -

libraries = [ 'niftiio', 'znz', 'z' ],

- -

5. Things to know

-

When accessing NIfTI image data through NumPy arrays the order of the -dimensions is reversed. If the x, y, z, t dimensions of a NIfTI image -are 64, 64, 32, 456 (as for example reported by nifti_tool), the shape -of the NumPy array (e.g. as returned by NiftiImage.asarray()) will -be: 456, 32, 64, 64.

-

This is done to be able to slice the data array much easier in the most -common cases. For example, if you are interested in a certain volume of a timeseries -it is much easier to write data[2] instead of -data[:,:,:,2], right?. - -

6. Examples

-

The next sections contains some examples showing ways to use PyNIfTI to -read and write imaging data from within Python to be able to process it with -some random Python library.

-

All examples assume that you have imported the PyNIfTI module by invoking:

-

from nifti import *

- -

a) Fileformat conversion

-

Open the MNI standard space template that is shipped with FSL. No filename -extension is necessary as libniftiio determines it automatically:

- -

nim = NiftiImage('avg152T1_brain')

- -

The filename is available via the 'filename' attribute:

-

print nim.filename

-

yields 'avg152T1_brain.img'. This indicates an ANALYZE image. If you want to -save this image as a single gzipped NIfTI file simply do:

-

nim.save('mni.nii.gz')

-

The filetype is determined from the filename. If you want to save to gzipped -ANALYZE file pairs instead the following would be an alternative to calling the -save() with a new filename.

-

nim.filename = 'mni_analyze.img.gz'
nim.save()

-

Please see the docstring of the NiftiImage.setFilename() method -to learn how the filetypes are determined from the filenames.

- -

b) NIfTI files from array data

-

The next code snipped demonstrates how to create a 4d NIfTI image containing -gaussian noise. First we need to import the NumPy module

-

import numpy

-

Now generate the noise dataset. Let's generate noise for 100 volumes with 16 -slices and a 32x32 inplane matrix.

-

noise = numpy.random.randn(100,16,32,32)

-

Please notice the order in which the dimensions are specified: -(t, z, y, x).

-

The datatype of the array will most likely be float64 -- which can -be verified by invoking noise.dtype.

-

Converting this dataset into a NIfTI image is done by invoking the -NiftiImage constructor with the noise dataset as argument:

-

nim = NiftiImage(noise)

- -

The relevant header information is extracted from the NumPy array. If you -query the header information about the dimensionality of the image, it returns -the desired values:

-

print nim.header['dim'] # yields: [4, 32, 32, 16, 100, 0, 0, 0]

-First value shows the number of dimensions in the datset: 4 (good, that's what -we wanted). The following numbers are dataset size on the x, y, z, t, u, v, w -axis (NIfTI files can handle up to 7 dimensions). Please notice, that the order -of dimensions is now 'correct': We have 32x32 inplane resolution, 16 slices in z -direction and 100 volumes.

-

Also the datatype was set appropriately. The exprression:

-

nim.header['datatype'] == nifticlib.NIFTI_TYPE_FLOAT64

-

will evaluate to True.

-

To save the noise file to disk, just call the save() method:

-

nim.save('noise.nii.gz')

- - -

c) Select ROIs

-

Suppose you want to have the first ten volumes of the noise dataset we have -just created in a separate file. First open the file (can be skipped if it is -still open):

-

nim = NiftiImage('noise.nii.gz')

-

Now select the first ten volumes and store them to another file, while -preserving as much header information as possible:

-

-nim2 = NiftiImage(nim.data[:10], nim.header)
-nim2.save('part.hdr.gz') -

-

The NiftiImage constructor takes a dictionary with header information as an -optional argument. Settings that are not determined by the array (e.g. size, -datatype) are copied from the dictionary and stored to the new NIfTI image.

- - -

d) Linear detrending of timeseries (SciPy module is required for this -example)

-

Let's load another 4d NIfTI file and perform a linear detrending, by fitting -a straight line to the timeseries of each voxel and substract that fit from the -data. Although this might sound complicated at first, thanks to the excellent -SciPy module it is just a few lines of code.

-

nim = NiftiImage('timeseries.nii')

-

Depending on the datatype of the input image the detrending process might -change the datatype from integer to float. As operations that change the -(binary) size of the NIfTI image are not supported, we need to make a copy -of the data and later create a new NIfTI image.

-

data = nim.asarray()

-

Now detrend the data along the time axis. Remember that the array has the -time axis as its first dimension (in contrast to the NIfTI file where it is -the 4th).

-

-from scipy import signal
-data_detrended = signal.detrend( data, axis=0 )

-

Finally, create a new NIfTI image using header information from the original -source image.

-

nim_detrended = NiftiImage( data_detrended, nim.header)

- -

e) Make a quick plot of a voxels timeseries (Gnuplot module is required)

-

Plotting is essential to get a 'feeling' for the data. The -python interface -to Gnuplot makes it really easy to plot -something (e.g. when running Python -interactively via IPython). Please -note, that there are many other possibilities for plotting. Some examples are: -using R via -RPy or Matlab-style plotting via -matplotlib.

-

However, using Gnuplot is really easy. First import the Gnuplot module -and create the interface object.

-

-from Gnuplot import Gnuplot
-gp = Gnuplot()

-

We want the timeseries as a line plot and not just the datapoints, so -let's talk with Gnuplot.

-

gp('set data style lines')

-

Now load a 4d NIfTI image

-

nim = NiftiImage('perfect_subject.nii.gz')

-

and finally plot the timeseries of voxel (x=20, y=30, z=12):

-

gp.plot(nim.data[:,12,30,20])

-

A Gnuplot window showing the timeseries should popup now (screenshot). Please read the -Gnuplot Manual to learn what it can do -- and it can do a lot more than just -simple line plots (have a look at this -page if you are interested).

- -

f) Show a slice of a 3d volume (Matplotlib module is required)

-

This example demonstrates howto use the Matlab-style plotting of Matplotlib to view a slice from a 3d volume.

-

This time I assume that a 3d nifti file is already opened and available in the nim3d object. At first we need to load the necessary Python module.

-

from pylab import *

-

If everything went fine, we can now view a slice (x,y):

-

imshow(nim3d.data[200], interpolation='nearest', cmap=cm.gray)
-show()

-

It is necessary to call the show() function one time after importing pylab to actually see the image when running Python interactively (screenshot).

When you want to have a look at a yz-slice, NumPy array magic comes into play.

-

imshow(nim3d.data[::-1,:,100], interpolation='nearest', cmap=cm.gray)

-

The ::-1 notation causes the z-axis to be flipped in the images. This makes a much nicer screenshot, because the used example volume has the z-axis originally oriented upsidedown.

- -

g) Compute and display peristimulus signal timecourse of multiple conditions with pynifti_pst and FSLView

-

Sometimes one wants to look at the signal timecourse of some voxel after a certain stimulation onset. An easy way would be to have some fMRI data viewer that displays a statistical map and one could click on some activated voxel and the peristimulus signal timecourse of some condition in that voxel would be displayed.

-

This can easily be done by using pynifti_pst and FSLView.

-

pynifti_pst comes with a manpage that explains all options and arguments. Basically pynifti_pst need a 4d image (e.g. an fMRI timeseries; possibly preprocessed/filtered) and some stimulus onset information. This information can either be given directly on the command line or is read from files. Additionally one can specify onsets as volume numbers or as onset times.

-

pynifti_pst understands the FSL custom EV file format so one can easily use those files as input.

-

An example call could look like this:

-

pynifti_pst --times --nvols 5 -p uf92.feat/filtered_func_data.nii.gz pst_cond_a.nii.gz uf92.feat/custom_timing_files/ev1.txt uf92.feat/custom_timing_files/ev2.txt

-

This computes a peristimulus timeseries using the preprocessed fMRI from a FEAT output directory -and two custom EV files that both together make up condition A. --times indicates that -the EV files list onset times (not volume ids) and --nvols requests the mean peristimulus -timecourse for 4 volumes after stimulus onset (5 including onset). -p recodes the -peristimulus timeseries into percent signalchange, where the onset is always zero and any following -value is the signal change with respect to the onset volume.

-

This call produces a simple 4d NIfTI image that can be loaded into FSLView as any other timeseries. The following call can be used to display an FSL zmap from the above results path on top of some anatomy. Additionally the peristimulus timeseries of two conditions are loaded. This screenshot shows how it could look like. One of the nice features of FSLView is that its timeseries window can remember selected curves, which can be useful to compare signal timecourses from different voxels (blue and green line in the screenshot).

-

fslview pst_cond_a.nii.gz pst_cond_b.nii.gz uf92_ana.nii.gz uf92.feat/stats/zstat1.nii.gz -b 3,5

- -

History

-

The full changelog is here.

- - From scipy-svn at scipy.org Thu Oct 4 01:30:15 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 00:30:15 -0500 (CDT) Subject: [Scipy-svn] r3400 - trunk/scipy/io/nifti Message-ID: <20071004053015.E3A3EC7C044@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 00:30:09 -0500 (Thu, 04 Oct 2007) New Revision: 3400 Added: trunk/scipy/io/nifti/__init__.py trunk/scipy/io/nifti/nifticlib.i trunk/scipy/io/nifti/nifticlib.py trunk/scipy/io/nifti/niftiimage.py trunk/scipy/io/nifti/utils.py Removed: trunk/scipy/io/nifti/nifti/ Log: simplify directory structure Copied: trunk/scipy/io/nifti/__init__.py (from rev 3399, trunk/scipy/io/nifti/nifti/__init__.py) Copied: trunk/scipy/io/nifti/nifticlib.i (from rev 3399, trunk/scipy/io/nifti/nifti/nifticlib.i) Copied: trunk/scipy/io/nifti/nifticlib.py (from rev 3399, trunk/scipy/io/nifti/nifti/nifticlib.py) Copied: trunk/scipy/io/nifti/niftiimage.py (from rev 3399, trunk/scipy/io/nifti/nifti/niftiimage.py) Copied: trunk/scipy/io/nifti/utils.py (from rev 3399, trunk/scipy/io/nifti/nifti/utils.py) From scipy-svn at scipy.org Thu Oct 4 01:46:20 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 00:46:20 -0500 (CDT) Subject: [Scipy-svn] r3401 - trunk/scipy/io/nifti Message-ID: <20071004054620.500CD39C1EA@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 00:46:17 -0500 (Thu, 04 Oct 2007) New Revision: 3401 Modified: trunk/scipy/io/nifti/setup.py Log: update setup.py to conform with SciPy Modified: trunk/scipy/io/nifti/setup.py =================================================================== --- trunk/scipy/io/nifti/setup.py 2007-10-04 05:30:09 UTC (rev 3400) +++ trunk/scipy/io/nifti/setup.py 2007-10-04 05:46:17 UTC (rev 3401) @@ -1,55 +1,51 @@ #!/usr/bin/env python - -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -# -# Python distutils setup for PyNifti -# -# Copyright (C) 2006-2007 by -# Michael Hanke -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the MIT License. -# -# This package is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING -# file that comes with this package for more details. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### - -from distutils.core import setup, Extension import os +from os.path import isfile, join, dirname +import sys import numpy -from glob import glob -nifti_wrapper_file = os.path.join('nifti', 'nifticlib.py') +nifti_wrapper_file = join('nifticlib.py') # create an empty file to workaround crappy swig wrapper installation -if not os.path.isfile(nifti_wrapper_file): +if not isfile(nifti_wrapper_file): open(nifti_wrapper_file, 'w') # find numpy headers -numpy_headers = os.path.join(os.path.dirname(numpy.__file__),'core','include') +numpy_headers = join(dirname(numpy.__file__),'core','include') +def configuration(parent_package='',top_path=None): + from numpy.distutils.misc_util import Configuration + from numpy.distutils.system_info import get_info -# Notes on the setup -# Version scheme is: -# 0.<4-digit-year><2-digit-month><2-digit-day>. + config = Configuration('nifti',parent_package,top_path) + config.add_data_dir('tests') -setup(name = 'pynifti', - version = '0.20070930.1', - author = 'Michael Hanke', - author_email = 'michael.hanke at gmail.com', - license = 'MIT License', - url = 'http://apsy.gse.uni-magdeburg.de/hanke', - description = 'Python interface for the NIfTI IO libraries', - long_description = """ """, - packages = [ 'nifti' ], - scripts = glob( 'bin/*' ), - ext_modules = [ Extension( 'nifti._nifticlib', [ 'nifti/nifticlib.i' ], - include_dirs = [ '/usr/include/nifti', numpy_headers ], - libraries = [ 'niftiio' ], - swig_opts = [ '-I/usr/include/nifti', - '-I' + numpy_headers ] ) ] - ) + include_dirs = [ + '.', + './nifticlib/fsliolib', + './nifticlib/niftilib', + './nifticlib/znzlib'] + nifticlib_headers = ' -I'.join(include_dirs) + swig_opts = ['-I'+nifticlib_headers, '-I'+numpy_headers] + + # Libraries + config.add_library('fslio', + sources=['./nifticlib/fsliolib/fslio.c'], include_dirs=include_dirs) + config.add_library('niftiio', + sources=['./nifticlib/niftilib/nifti1_io.c'], include_dirs=include_dirs) + config.add_library('znz', + sources=['./nifticlib/znzlib/znzlib.c'], include_dirs=include_dirs) + + # Extension + config.add_extension('_nifticlib', + sources = ['nifticlib.i'], + include_dirs = include_dirs, + libraries = ['niftiio', 'fslio', 'znz',], + swig_opts = swig_opts) + + return config + +if __name__ == '__main__': + from numpy.distutils.core import setup + setup(**configuration(top_path='').todict()) From scipy-svn at scipy.org Thu Oct 4 01:54:37 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 00:54:37 -0500 (CDT) Subject: [Scipy-svn] r3402 - in trunk/scipy/io/nifti: . tests Message-ID: <20071004055437.1CAF939C083@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 00:54:33 -0500 (Thu, 04 Oct 2007) New Revision: 3402 Modified: trunk/scipy/io/nifti/__init__.py trunk/scipy/io/nifti/tests/test_fileio.py trunk/scipy/io/nifti/tests/test_utils.py trunk/scipy/io/nifti/utils.py Log: update imports Modified: trunk/scipy/io/nifti/__init__.py =================================================================== --- trunk/scipy/io/nifti/__init__.py 2007-10-04 05:46:17 UTC (rev 3401) +++ trunk/scipy/io/nifti/__init__.py 2007-10-04 05:54:33 UTC (rev 3402) @@ -15,4 +15,4 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -from niftiimage import * +from scipy.io.nifti.niftiimage import * Modified: trunk/scipy/io/nifti/tests/test_fileio.py =================================================================== --- trunk/scipy/io/nifti/tests/test_fileio.py 2007-10-04 05:46:17 UTC (rev 3401) +++ trunk/scipy/io/nifti/tests/test_fileio.py 2007-10-04 05:54:33 UTC (rev 3402) @@ -15,13 +15,13 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -import nifti import unittest import md5 import tempfile import shutil import os import numpy as np +import scipy.io.nifti def md5sum(filename): Modified: trunk/scipy/io/nifti/tests/test_utils.py =================================================================== --- trunk/scipy/io/nifti/tests/test_utils.py 2007-10-04 05:46:17 UTC (rev 3401) +++ trunk/scipy/io/nifti/tests/test_utils.py 2007-10-04 05:54:33 UTC (rev 3402) @@ -15,7 +15,7 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -import nifti.utils +import scipy.io.nifti.utils import numpy import unittest Modified: trunk/scipy/io/nifti/utils.py =================================================================== --- trunk/scipy/io/nifti/utils.py 2007-10-04 05:46:17 UTC (rev 3401) +++ trunk/scipy/io/nifti/utils.py 2007-10-04 05:54:33 UTC (rev 3402) @@ -15,8 +15,8 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -import nifti import numpy +import scipy.io.nifti def time2vol( t, tr, lag=0.0, decimals=0 ): """ Translates a time 't' into a volume number. By default function returns From scipy-svn at scipy.org Thu Oct 4 02:00:07 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 01:00:07 -0500 (CDT) Subject: [Scipy-svn] r3403 - trunk/scipy/io/nifti Message-ID: <20071004060007.670A639C083@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 01:00:05 -0500 (Thu, 04 Oct 2007) New Revision: 3403 Modified: trunk/scipy/io/nifti/niftiimage.py Log: remove decorators Modified: trunk/scipy/io/nifti/niftiimage.py =================================================================== --- trunk/scipy/io/nifti/niftiimage.py 2007-10-04 05:54:33 UTC (rev 3402) +++ trunk/scipy/io/nifti/niftiimage.py 2007-10-04 06:00:05 UTC (rev 3403) @@ -52,7 +52,6 @@ } - @staticmethod def numpydtype2niftidtype(array): """ Return the NIfTI datatype id for a corrsponding numpy array datatype. @@ -64,9 +63,8 @@ raise ValueError, "Unsupported datatype '%s'" % str(array.dtype) return NiftiImage.numpy2nifti_dtype_map[dtype] + numpydtype2niftidtype = staticmethod(numpydtype2niftidtype) - - @staticmethod def splitFilename(filename): """ Split a NIfTI filename and returns a tuple of basename and extension. If no valid NIfTI filename extension is found, the whole @@ -85,9 +83,8 @@ return filename, '' else: return '.'.join(parts[:-1]), parts[-1] + splitFilename = staticmethod(splitFilename) - - @staticmethod def nhdr2dict(nhdr): """ Convert a NIfTI header struct into a python dictionary. @@ -138,9 +135,8 @@ h['qoffset'] = [ nhdr.qoffset_x, nhdr.qoffset_y, nhdr.qoffset_z ] return h + nhdr2dict = staticmethod(nhdr2dict) - - @staticmethod def updateNiftiHeaderFromDict(nhdr, hdrdict): """ Update a NIfTI header struct with data from a dictionary. @@ -289,8 +285,8 @@ raise ValueError, \ "Nifti header property 'magic' must be 'ni1' or 'n+1'." nhdr.magic = hdrdict['magic'] + updateNiftiHeaderFromDict = staticmethod(updateNiftiHeaderFromDict) - def __init__(self, source, header = {}, load=False ): """ Create a Niftifile object. From scipy-svn at scipy.org Thu Oct 4 03:05:20 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 02:05:20 -0500 (CDT) Subject: [Scipy-svn] r3404 - trunk/scipy/io Message-ID: <20071004070520.AB6D039C14A@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 02:05:17 -0500 (Thu, 04 Oct 2007) New Revision: 3404 Modified: trunk/scipy/io/__init__.py trunk/scipy/io/setup.py Log: adding nifti to scipy.io setup and init Modified: trunk/scipy/io/__init__.py =================================================================== --- trunk/scipy/io/__init__.py 2007-10-04 06:00:05 UTC (rev 3403) +++ trunk/scipy/io/__init__.py 2007-10-04 07:05:17 UTC (rev 3404) @@ -16,6 +16,7 @@ from array_import import * from data_store import * from pickler import * +from scipy.io.nifti import NiftiImage from mmio import mminfo,mmread,mmwrite Modified: trunk/scipy/io/setup.py =================================================================== --- trunk/scipy/io/setup.py 2007-10-04 06:00:05 UTC (rev 3403) +++ trunk/scipy/io/setup.py 2007-10-04 07:05:17 UTC (rev 3404) @@ -3,8 +3,12 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration('io', parent_package, top_path) + + config.add_subpackage('nifti') + config.add_extension('numpyio', sources = ['numpyiomodule.c']) + config.add_data_dir('tests') config.add_data_dir('examples') config.add_data_dir('docs') From scipy-svn at scipy.org Thu Oct 4 03:19:58 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 02:19:58 -0500 (CDT) Subject: [Scipy-svn] r3405 - trunk/scipy/io/nifti Message-ID: <20071004071958.7DE7E39C04F@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 02:19:55 -0500 (Thu, 04 Oct 2007) New Revision: 3405 Removed: trunk/scipy/io/nifti/bin/ trunk/scipy/io/nifti/man/ trunk/scipy/io/nifti/utils.py Log: remove script Deleted: trunk/scipy/io/nifti/utils.py =================================================================== --- trunk/scipy/io/nifti/utils.py 2007-10-04 07:05:17 UTC (rev 3404) +++ trunk/scipy/io/nifti/utils.py 2007-10-04 07:19:55 UTC (rev 3405) @@ -1,129 +0,0 @@ -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -# -# Utility function for PyNifti -# -# Copyright (C) 2007 by -# Michael Hanke -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the MIT License. -# -# This package is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING -# file that comes with this package for more details. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## - -import numpy -import scipy.io.nifti - -def time2vol( t, tr, lag=0.0, decimals=0 ): - """ Translates a time 't' into a volume number. By default function returns - the volume number that is closest in time. Volumes are assumed to be - recorded exactly (and completely) after tr/2, e.g. if 'tr' is 2 secs the - first volume is recorded at exactly one second. - - 't' might be a single value, a sequence or an array. - - The repetition 'tr' might be specified directly, but can also be a - NiftiImage object. In the latter case the value of 'tr' is determined from - the 'rtime' property of the NiftiImage object. - - 't' and 'tr' can be given in an arbitrary unit (but both have to be in the - same unit). - - The 'lag' argument can be used to shift the times by constant offset. - - Please note that numpy.round() is used to round to interger value (rounds - to even numbers). The 'decimals' argument will be passed to numpy.round(). - """ - # transform to numpy array for easy handling - tmp = numpy.array(t) - - # determine tr if NiftiImage object - if isinstance( tr, nifti.NiftiImage ): - tr = tr.rtime - - vol = numpy.round( ( tmp + lag + tr/2 ) / tr, decimals ) - - return vol - - -def applyFxToVolumes( ts, vols, fx, **kwargs ): - """ Apply a function on selected volumes of a timeseries. - - 'ts' is a 4d timeseries. It can be a NiftiImage or a numpy array. - In case of a numpy array one has to make sure that the time is on the - first axis. 'ts' can actually be of any dimensionality, but datasets aka - volumes are assumed to be along the first axis. - - 'vols' is either a sequence of sequences or a 2d array indicating which - volumes fx should be applied to. Each row defines a set of volumes. - - 'fx' is a callable function to get an array of the selected volumes as - argument. Additonal arguments may be specified as keyword arguments and - are passed to 'fx'. - - The output will be a 4d array with one computed volume per row in the 'vols' - array. - """ - # get data array from nifti image or assume data array is - # already present - if isinstance( ts, nifti.NiftiImage ): - data = ts.data - else: - data = ts - - out = [] - - for vol in vols: - out.append( fx( data[ numpy.array( vol ) ], **kwargs ) ) - - return numpy.array( out ) - - -def cropImage( nimg, bbox ): - """ Crop an image. - - 'bbox' has to be a sequency of (min,max) tuples (one for each image - dimension). - - The function returns the cropped image. The data is not shared with the - original image, but is copied. - """ - - # build crop command - cmd = 'nimg.data.squeeze()[' - cmd += ','.join( [ ':'.join( [ str(i) for i in dim ] ) for dim in bbox ] ) - cmd += ']' - - # crop the image data array - cropped = eval(cmd).copy() - - # return the cropped image with preserved header data - return nifti.NiftiImage(cropped, nimg.header) - - -def getPeristimulusTimeseries( ts, onsetvols, nvols = 10, fx = numpy.mean ): - """ Returns 4d array with peristimulus timeseries. - - Parameters: - ts - source 4d timeseries - onsetvols - sequence of onsetvolumes to be averaged over - nvols - length of the peristimulus timeseries in volumes - (starting from onsetvol) - fx - function to be applied to the list of corresponding - volumes. Typically this will be mean(), so it is default, - but it could also be var() or something different. The - supplied function is to be able to handle an 'axis=0' - argument similiar to NumPy's mean(), var(), ... - """ - selected = [ [ o + offset for o in onsetvols ] \ - for offset in range( nvols ) ] - - if fx == tuple: - return applyFxToVolumes( ts, selected, fx ) - else: - return applyFxToVolumes( ts, selected, fx, axis=0 ) - From scipy-svn at scipy.org Thu Oct 4 03:23:18 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 02:23:18 -0500 (CDT) Subject: [Scipy-svn] r3406 - trunk/scipy/io/nifti Message-ID: <20071004072318.84EED39C201@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 02:23:16 -0500 (Thu, 04 Oct 2007) New Revision: 3406 Modified: trunk/scipy/io/nifti/AUTHOR Log: updated authors of nifti io code Modified: trunk/scipy/io/nifti/AUTHOR =================================================================== --- trunk/scipy/io/nifti/AUTHOR 2007-10-04 07:19:55 UTC (rev 3405) +++ trunk/scipy/io/nifti/AUTHOR 2007-10-04 07:23:16 UTC (rev 3406) @@ -1,2 +1,5 @@ Michael Hanke +modified for SciPy by +Jarrod Millman +Chris Burns From scipy-svn at scipy.org Thu Oct 4 03:29:15 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 02:29:15 -0500 (CDT) Subject: [Scipy-svn] r3407 - trunk/scipy/io/nifti Message-ID: <20071004072915.A7E6639C04F@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 02:29:13 -0500 (Thu, 04 Oct 2007) New Revision: 3407 Removed: trunk/scipy/io/nifti/MANIFEST.in trunk/scipy/io/nifti/Makefile trunk/scipy/io/nifti/PKG-INFO Log: removing unneeded files Deleted: trunk/scipy/io/nifti/MANIFEST.in =================================================================== --- trunk/scipy/io/nifti/MANIFEST.in 2007-10-04 07:23:16 UTC (rev 3406) +++ trunk/scipy/io/nifti/MANIFEST.in 2007-10-04 07:29:13 UTC (rev 3407) @@ -1,5 +0,0 @@ -include AUTHOR COPYING Makefile README.html MANIFEST.in -include Changelog TODO -include man/* -include tests/* tests/data/* -prune nifti/wrap.py Deleted: trunk/scipy/io/nifti/Makefile =================================================================== --- trunk/scipy/io/nifti/Makefile 2007-10-04 07:23:16 UTC (rev 3406) +++ trunk/scipy/io/nifti/Makefile 2007-10-04 07:29:13 UTC (rev 3407) @@ -1,43 +0,0 @@ -all: - -distclean: - -rm MANIFEST Changelog - -rm nifti/*.{c,pyc,so} nifti/nifticlib.py - -rm tests/*.pyc - -rm -r build - -rm -r dist - - -orig-src: distclean - # clean existing dist dir first to have a single source tarball to process - -rm -rf dist - # the debian changelog is also the upstream changelog - cp debian/changelog Changelog - - # update manpages - help2man -N -n "compute peristimulus timeseries of fMRI data" \ - bin/pynifti_pst > man/pynifti_pst.1 - - if [ ! "$$(dpkg-parsechangelog | egrep ^Version | cut -d ' ' -f 2,2 | cut -d '-' -f 1,1)" == "$$(python setup.py -V)" ]; then \ - printf "WARNING: Changelog version does not match tarball version!\n" ;\ - exit 1; \ - fi - # let python create the source tarball - python setup.py sdist --formats=gztar - # rename to proper Debian orig source tarball and move upwards - # to keep it out of the Debian diff - file=$$(ls -1 dist); ver=$${file%*.tar.gz}; ver=$${ver#pynifti-*}; mv dist/$$file ../pynifti_$$ver.orig.tar.gz - -bdist_wininst: - # THIS IS ONLY FOR WINDOWS! - # Consider this a set of notes on how to build PyNIfTI on win32, rather - # than an actually working target - # - # assumes Dev-Cpp to be installed at C:\Dev-Cpp - python setup.py build_ext -c mingw32 --swig-opts "-C:\Dev-Cpp\include/nifti -DWIN32" -IC:\Dev-Cpp\include nifti - - # for some stupid reason the swig wrapper is in the wrong location - move /Y nifticlib.py nifti - - # now build the installer - python setup.py bdist_wininst Deleted: trunk/scipy/io/nifti/PKG-INFO =================================================================== --- trunk/scipy/io/nifti/PKG-INFO 2007-10-04 07:23:16 UTC (rev 3406) +++ trunk/scipy/io/nifti/PKG-INFO 2007-10-04 07:29:13 UTC (rev 3407) @@ -1,10 +0,0 @@ -Metadata-Version: 1.0 -Name: pynifti -Version: 0.20070930.1 -Summary: Python interface for the NIfTI IO libraries -Home-page: http://apsy.gse.uni-magdeburg.de/hanke -Author: Michael Hanke -Author-email: michael.hanke at gmail.com -License: MIT License -Description: -Platform: UNKNOWN From scipy-svn at scipy.org Thu Oct 4 03:33:34 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 02:33:34 -0500 (CDT) Subject: [Scipy-svn] r3408 - trunk/scipy/io/nifti/tests Message-ID: <20071004073334.CABEB39C04F@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 02:33:32 -0500 (Thu, 04 Oct 2007) New Revision: 3408 Added: trunk/scipy/io/nifti/tests/test_nifti.py Removed: trunk/scipy/io/nifti/tests/test_fileio.py trunk/scipy/io/nifti/tests/test_main.py trunk/scipy/io/nifti/tests/test_utils.py Log: starting to integrate tests with NumpyTest Deleted: trunk/scipy/io/nifti/tests/test_fileio.py =================================================================== --- trunk/scipy/io/nifti/tests/test_fileio.py 2007-10-04 07:29:13 UTC (rev 3407) +++ trunk/scipy/io/nifti/tests/test_fileio.py 2007-10-04 07:33:32 UTC (rev 3408) @@ -1,81 +0,0 @@ -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -# -# Unit tests for PyNIfTI file io -# -# Copyright (C) 2007 by -# Michael Hanke -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the MIT License. -# -# This package is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING -# file that comes with this package for more details. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### - -import unittest -import md5 -import tempfile -import shutil -import os -import numpy as np -import scipy.io.nifti - - -def md5sum(filename): - """ Generate MD5 hash string. - """ - file = open( filename ) - sum = md5.new() - while True: - data = file.read() - if not data: - break - sum.update(data) - return sum.hexdigest() - - -class FileIOTests(unittest.TestCase): - def setUp(self): - self.workdir = tempfile.mkdtemp('pynifti_test') - - def tearDown(self): - shutil.rmtree(self.workdir) - - def testIdempotentLoadSaveCycle(self): - """ check if file is unchanged by load/save cycle. - """ - md5_orig = md5sum('data/example4d.nii.gz') - nimg = nifti.NiftiImage('data/example4d.nii.gz') - nimg.save( os.path.join( self.workdir, 'iotest.nii.gz') ) - md5_io = md5sum( os.path.join( self.workdir, 'iotest.nii.gz') ) - - self.failUnlessEqual(md5_orig, md5_io) - - def testQFormSetting(self): - nimg = nifti.NiftiImage('data/example4d.nii.gz') - # 4x4 identity matrix - ident = np.identity(4) - self.failIf( (nimg.qform == ident).all() ) - - # assign new qform - nimg.qform = ident - self.failUnless( (nimg.qform == ident).all() ) - - # test save/load cycle - nimg.save( os.path.join( self.workdir, 'qformtest.nii.gz') ) - nimg2 = nifti.NiftiImage( os.path.join( self.workdir, - 'qformtest.nii.gz') ) - - self.failUnless( (nimg.qform == nimg2.qform).all() ) - - -def suite(): - return unittest.makeSuite(FileIOTests) - - -if __name__ == '__main__': - unittest.main() - Deleted: trunk/scipy/io/nifti/tests/test_main.py =================================================================== --- trunk/scipy/io/nifti/tests/test_main.py 2007-10-04 07:29:13 UTC (rev 3407) +++ trunk/scipy/io/nifti/tests/test_main.py 2007-10-04 07:33:32 UTC (rev 3408) @@ -1,42 +0,0 @@ -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -# -# Main unit test interface for PyNIfTI -# -# Copyright (C) 2007 by -# Michael Hanke -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the MIT License. -# -# This package is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING -# file that comes with this package for more details. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### - -import unittest - -# list all test modules (without .py extension) -tests = [ 'test_fileio', - 'test_utils', - ] - - -# import all test modules -for t in tests: - exec 'import ' + t - - -if __name__ == '__main__': - - # load all tests suites - suites = [ eval(t + '.suite()') for t in tests ] - - # and make global test suite - ts = unittest.TestSuite( suites ) - - # finally run it - unittest.TextTestRunner().run( ts ) - - Copied: trunk/scipy/io/nifti/tests/test_nifti.py (from rev 3406, trunk/scipy/io/nifti/tests/test_fileio.py) Deleted: trunk/scipy/io/nifti/tests/test_utils.py =================================================================== --- trunk/scipy/io/nifti/tests/test_utils.py 2007-10-04 07:29:13 UTC (rev 3407) +++ trunk/scipy/io/nifti/tests/test_utils.py 2007-10-04 07:33:32 UTC (rev 3408) @@ -1,37 +0,0 @@ -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -# -# Unit tests for PyNIfTI file io -# -# Copyright (C) 2007 by -# Michael Hanke -# -# This is free software; you can redistribute it and/or -# modify it under the terms of the MIT License. -# -# This package is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the COPYING -# file that comes with this package for more details. -# -### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### - -import scipy.io.nifti.utils -import numpy -import unittest - - -class UtilsTests(unittest.TestCase): - def testZScoring(self): - # dataset: mean=2, std=1 - data = numpy.array( (0,1,3,4,2,2,3,1,1,3,3,1,2,2,2,2) ) - self.failUnlessEqual( data.mean(), 2.0 ) - self.failUnlessEqual( data.std(), 1.0 ) - - -def suite(): - return unittest.makeSuite(UtilsTests) - - -if __name__ == '__main__': - unittest.main() - From scipy-svn at scipy.org Thu Oct 4 03:51:45 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 02:51:45 -0500 (CDT) Subject: [Scipy-svn] r3409 - trunk/scipy/io/nifti/tests Message-ID: <20071004075145.B9EECC7C053@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 02:51:43 -0500 (Thu, 04 Oct 2007) New Revision: 3409 Modified: trunk/scipy/io/nifti/tests/test_nifti.py Log: still working on tests Modified: trunk/scipy/io/nifti/tests/test_nifti.py =================================================================== --- trunk/scipy/io/nifti/tests/test_nifti.py 2007-10-04 07:33:32 UTC (rev 3408) +++ trunk/scipy/io/nifti/tests/test_nifti.py 2007-10-04 07:51:43 UTC (rev 3409) @@ -19,9 +19,10 @@ import md5 import tempfile import shutil -import os +from os.path import join + import numpy as np -import scipy.io.nifti +from scipy.io.nifti import NiftiImage def md5sum(filename): @@ -37,25 +38,25 @@ return sum.hexdigest() -class FileIOTests(unittest.TestCase): +class TestNiftiImage(unittest.TestCase): def setUp(self): self.workdir = tempfile.mkdtemp('pynifti_test') def tearDown(self): shutil.rmtree(self.workdir) - def testIdempotentLoadSaveCycle(self): + def test_idempotent_load_save_cycle(self): """ check if file is unchanged by load/save cycle. """ md5_orig = md5sum('data/example4d.nii.gz') - nimg = nifti.NiftiImage('data/example4d.nii.gz') - nimg.save( os.path.join( self.workdir, 'iotest.nii.gz') ) - md5_io = md5sum( os.path.join( self.workdir, 'iotest.nii.gz') ) + nimg = NiftiImage('data/example4d.nii.gz') + nimg.save( join( self.workdir, 'iotest.nii.gz') ) + md5_io = md5sum( join( self.workdir, 'iotest.nii.gz') ) self.failUnlessEqual(md5_orig, md5_io) - def testQFormSetting(self): - nimg = nifti.NiftiImage('data/example4d.nii.gz') + def test_qform_setting(self): + nimg = NiftiImage('data/example4d.nii.gz') # 4x4 identity matrix ident = np.identity(4) self.failIf( (nimg.qform == ident).all() ) @@ -65,17 +66,12 @@ self.failUnless( (nimg.qform == ident).all() ) # test save/load cycle - nimg.save( os.path.join( self.workdir, 'qformtest.nii.gz') ) - nimg2 = nifti.NiftiImage( os.path.join( self.workdir, - 'qformtest.nii.gz') ) + nimg.save( join( self.workdir, 'qformtest.nii.gz') ) + nimg2 = NiftiImage( join( self.workdir, 'qformtest.nii.gz') ) self.failUnless( (nimg.qform == nimg2.qform).all() ) -def suite(): - return unittest.makeSuite(FileIOTests) - - if __name__ == '__main__': - unittest.main() + NumpyTest.run() From scipy-svn at scipy.org Thu Oct 4 04:15:43 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 03:15:43 -0500 (CDT) Subject: [Scipy-svn] r3410 - trunk/scipy/io/nifti Message-ID: <20071004081543.B3152C7C045@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 03:15:40 -0500 (Thu, 04 Oct 2007) New Revision: 3410 Modified: trunk/scipy/io/nifti/__init__.py trunk/scipy/io/nifti/niftiimage.py trunk/scipy/io/nifti/setup.py Log: minor cleanups Modified: trunk/scipy/io/nifti/__init__.py =================================================================== --- trunk/scipy/io/nifti/__init__.py 2007-10-04 07:51:43 UTC (rev 3409) +++ trunk/scipy/io/nifti/__init__.py 2007-10-04 08:15:40 UTC (rev 3410) @@ -15,4 +15,4 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### -from scipy.io.nifti.niftiimage import * +from scipy.io.nifti.niftiimage import NiftiImage Modified: trunk/scipy/io/nifti/niftiimage.py =================================================================== --- trunk/scipy/io/nifti/niftiimage.py 2007-10-04 07:51:43 UTC (rev 3409) +++ trunk/scipy/io/nifti/niftiimage.py 2007-10-04 08:15:40 UTC (rev 3410) @@ -16,9 +16,8 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # the swig wrapper if the NIfTI C library -import nifticlib -import os import numpy +import scipy.io.nifticlib as nifticlib class NiftiImage(object): Modified: trunk/scipy/io/nifti/setup.py =================================================================== --- trunk/scipy/io/nifti/setup.py 2007-10-04 07:51:43 UTC (rev 3409) +++ trunk/scipy/io/nifti/setup.py 2007-10-04 08:15:40 UTC (rev 3410) @@ -1,7 +1,6 @@ #!/usr/bin/env python -import os + from os.path import isfile, join, dirname -import sys import numpy nifti_wrapper_file = join('nifticlib.py') @@ -11,13 +10,12 @@ open(nifti_wrapper_file, 'w') # find numpy headers -numpy_headers = join(dirname(numpy.__file__),'core','include') +numpy_headers = join(dirname(numpy.__file__), 'core', 'include') -def configuration(parent_package='',top_path=None): +def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration - from numpy.distutils.system_info import get_info + config = Configuration('nifti', parent_package, top_path) - config = Configuration('nifti',parent_package,top_path) config.add_data_dir('tests') include_dirs = [ From scipy-svn at scipy.org Thu Oct 4 13:41:51 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 12:41:51 -0500 (CDT) Subject: [Scipy-svn] r3411 - trunk/scipy/io/nifti Message-ID: <20071004174151.C985C39C20E@new.scipy.org> Author: chris.burns Date: 2007-10-04 12:41:49 -0500 (Thu, 04 Oct 2007) New Revision: 3411 Modified: trunk/scipy/io/nifti/niftiimage.py Log: Fix nifticlib import error. Modified: trunk/scipy/io/nifti/niftiimage.py =================================================================== --- trunk/scipy/io/nifti/niftiimage.py 2007-10-04 08:15:40 UTC (rev 3410) +++ trunk/scipy/io/nifti/niftiimage.py 2007-10-04 17:41:49 UTC (rev 3411) @@ -17,9 +17,8 @@ # the swig wrapper if the NIfTI C library import numpy -import scipy.io.nifticlib as nifticlib +import nifticlib - class NiftiImage(object): """Wrapper class for convenient access to NIfTI data. From scipy-svn at scipy.org Thu Oct 4 13:43:53 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 12:43:53 -0500 (CDT) Subject: [Scipy-svn] r3412 - in trunk/scipy/sandbox/maskedarray: . tests Message-ID: <20071004174353.9D16B39C20A@new.scipy.org> Author: pierregm Date: 2007-10-04 12:43:51 -0500 (Thu, 04 Oct 2007) New Revision: 3412 Modified: trunk/scipy/sandbox/maskedarray/core.py trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py Log: core: fixed any test_subclassing : fixed _get_series Modified: trunk/scipy/sandbox/maskedarray/core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/core.py 2007-10-04 17:41:49 UTC (rev 3411) +++ trunk/scipy/sandbox/maskedarray/core.py 2007-10-04 17:43:51 UTC (rev 3412) @@ -1503,7 +1503,7 @@ def __idiv__(self, other): "Divides self by other in place." other_data = getdata(other) - dom_mask = domain_safe_divide().__call__(self._data, other_data) + dom_mask = _DomainSafeDivide().__call__(self._data, other_data) other_mask = getmask(other) new_mask = mask_or(other_mask, dom_mask) # The following 3 lines control the domain filling @@ -1700,14 +1700,14 @@ An exception is raised if ``out`` is not None and not of the same type as self. """ if out is None: - d = self.filled(True).any(axis=axis).view(type(self)) + d = self.filled(False).any(axis=axis).view(type(self)) if d.ndim > 0: d.__setmask__(self._mask.all(axis)) return d elif type(out) is not type(self): raise TypeError("The external array should have a type %s (got %s instead)" %\ (type(self), type(out))) - self.filled(True).any(axis=axis, out=out) + self.filled(False).any(axis=axis, out=out) if out.ndim: out.__setmask__(self._mask.all(axis)) return out @@ -2934,4 +2934,25 @@ # z = empty(3,) mmys.all(0, out=z) + + if 1: + x = numpy.array([[ 0.13, 0.26, 0.90], + [ 0.28, 0.33, 0.63], + [ 0.31, 0.87, 0.70]]) + m = numpy.array([[ True, False, False], + [False, False, False], + [True, True, False]], dtype=numpy.bool_) + mx = masked_array(x, mask=m) + xbig = numpy.array([[False, False, True], + [False, False, True], + [False, True, True]], dtype=numpy.bool_) + mxbig = (mx > 0.5) + mxsmall = (mx < 0.5) + # + assert (mxbig.all()==False) + assert (mxbig.any()==True) + assert_equal(mxbig.all(0),[False, False, True]) + assert_equal(mxbig.all(1), [False, False, True]) + assert_equal(mxbig.any(0),[False, False, True]) + assert_equal(mxbig.any(1), [True, True, True]) \ No newline at end of file Modified: trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py 2007-10-04 17:41:49 UTC (rev 3411) +++ trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py 2007-10-04 17:43:51 UTC (rev 3412) @@ -49,7 +49,9 @@ SubArray.__array_finalize__(self, obj) return def _get_series(self): - return self.view(MaskedArray) + _view = self.view(MaskedArray) + _view._sharedmask = False + return _view _series = property(fget=_get_series) msubarray = MSubArray @@ -63,7 +65,9 @@ MaskedArray.__array_finalize__(self,obj) return def _get_series(self): - return self.view(MaskedArray) + _view = self.view(MaskedArray) + _view._sharedmask = False + return _view _series = property(fget=_get_series) mmatrix = MMatrix From scipy-svn at scipy.org Thu Oct 4 15:34:47 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 14:34:47 -0500 (CDT) Subject: [Scipy-svn] r3413 - trunk/scipy/sandbox/maskedarray/tests Message-ID: <20071004193447.494EB39C0DF@new.scipy.org> Author: pierregm Date: 2007-10-04 14:34:43 -0500 (Thu, 04 Oct 2007) New Revision: 3413 Modified: trunk/scipy/sandbox/maskedarray/tests/test_core.py trunk/scipy/sandbox/maskedarray/tests/test_morestats.py trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py Log: tests : put the classes names in CamelCase Modified: trunk/scipy/sandbox/maskedarray/tests/test_core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_core.py 2007-10-04 17:43:51 UTC (rev 3412) +++ trunk/scipy/sandbox/maskedarray/tests/test_core.py 2007-10-04 19:34:43 UTC (rev 3413) @@ -27,7 +27,7 @@ pi = numpy.pi #.............................................................................. -class TestMa(NumpyTestCase): +class TestMA(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/maskedarray/tests/test_morestats.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_morestats.py 2007-10-04 17:43:51 UTC (rev 3412) +++ trunk/scipy/sandbox/maskedarray/tests/test_morestats.py 2007-10-04 19:34:43 UTC (rev 3413) @@ -3,12 +3,12 @@ :author: Pierre Gerard-Marchant :contact: pierregm_at_uga_dot_edu -:version: $Id: test_morestats.py 240 2007-07-09 15:36:48Z backtopop $ +:version: $Id: test_morestats.py 317 2007-10-04 19:31:14Z backtopop $ """ __author__ = "Pierre GF Gerard-Marchant ($Author: backtopop $)" __version__ = '1.0' -__revision__ = "$Revision: 240 $" -__date__ = '$Date: 2007-07-09 11:36:48 -0400 (Mon, 09 Jul 2007) $' +__revision__ = "$Revision: 317 $" +__date__ = '$Date: 2007-10-04 15:31:14 -0400 (Thu, 04 Oct 2007) $' import numpy Modified: trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py 2007-10-04 17:43:51 UTC (rev 3412) +++ trunk/scipy/sandbox/maskedarray/tests/test_mrecords.py 2007-10-04 19:34:43 UTC (rev 3413) @@ -30,7 +30,7 @@ fromarrays, fromtextfile, fromrecords, addfield #.............................................................................. -class TestMrecords(NumpyTestCase): +class TestMRecords(NumpyTestCase): "Base test class for MaskedArrays." def __init__(self, *args, **kwds): NumpyTestCase.__init__(self, *args, **kwds) Modified: trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py 2007-10-04 17:43:51 UTC (rev 3412) +++ trunk/scipy/sandbox/maskedarray/tests/test_subclassing.py 2007-10-04 19:34:43 UTC (rev 3413) @@ -65,7 +65,7 @@ MaskedArray.__array_finalize__(self,obj) return def _get_series(self): - _view = self.view(MaskedArray) + _view = self.view(MaskedArray) _view._sharedmask = False return _view _series = property(fget=_get_series) @@ -171,8 +171,15 @@ assert isinstance(z._data, SubArray) assert_equal(z._data.info, {}) # - z = ym+1 + z = (ym+1) + assert isinstance(z, MaskedArray) + assert isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert z._data.info['added'] > 0 + # + ym._set_mask([1,0,0,0,1]) + assert_equal(ym._mask, [1,0,0,0,1]) + ym._series._set_mask([0,0,0,0,1]) + assert_equal(ym._mask, [0,0,0,0,1]) - - From scipy-svn at scipy.org Thu Oct 4 17:22:51 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 16:22:51 -0500 (CDT) Subject: [Scipy-svn] r3414 - trunk/scipy/io/nifti Message-ID: <20071004212251.749A439C21B@new.scipy.org> Author: jarrod.millman Date: 2007-10-04 16:22:48 -0500 (Thu, 04 Oct 2007) New Revision: 3414 Modified: trunk/scipy/io/nifti/niftiimage.py Log: using an absolute import Modified: trunk/scipy/io/nifti/niftiimage.py =================================================================== --- trunk/scipy/io/nifti/niftiimage.py 2007-10-04 19:34:43 UTC (rev 3413) +++ trunk/scipy/io/nifti/niftiimage.py 2007-10-04 21:22:48 UTC (rev 3414) @@ -17,7 +17,7 @@ # the swig wrapper if the NIfTI C library import numpy -import nifticlib +from scipy.io.nifti import nifticlib class NiftiImage(object): """Wrapper class for convenient access to NIfTI data. From scipy-svn at scipy.org Thu Oct 4 17:59:28 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 4 Oct 2007 16:59:28 -0500 (CDT) Subject: [Scipy-svn] r3415 - trunk/scipy/io Message-ID: <20071004215928.76CAB39C25C@new.scipy.org> Author: chris.burns Date: 2007-10-04 16:59:25 -0500 (Thu, 04 Oct 2007) New Revision: 3415 Modified: trunk/scipy/io/__init__.py trunk/scipy/io/setup.py Log: comment out nifti until Windows build is resolved Modified: trunk/scipy/io/__init__.py =================================================================== --- trunk/scipy/io/__init__.py 2007-10-04 21:22:48 UTC (rev 3414) +++ trunk/scipy/io/__init__.py 2007-10-04 21:59:25 UTC (rev 3415) @@ -16,8 +16,11 @@ from array_import import * from data_store import * from pickler import * -from scipy.io.nifti import NiftiImage +# Comment out until we resolve nifti build on Windows. +#from scipy.io.nifti import NiftiImage +#del nifti + from mmio import mminfo,mmread,mmwrite __all__ = filter(lambda s:not s.startswith('_'),dir()) Modified: trunk/scipy/io/setup.py =================================================================== --- trunk/scipy/io/setup.py 2007-10-04 21:22:48 UTC (rev 3414) +++ trunk/scipy/io/setup.py 2007-10-04 21:59:25 UTC (rev 3415) @@ -4,7 +4,8 @@ from numpy.distutils.misc_util import Configuration config = Configuration('io', parent_package, top_path) - config.add_subpackage('nifti') + # Comment out until we resolve nifti build on Windows. + #config.add_subpackage('nifti') config.add_extension('numpyio', sources = ['numpyiomodule.c']) From scipy-svn at scipy.org Fri Oct 5 06:08:32 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 5 Oct 2007 05:08:32 -0500 (CDT) Subject: [Scipy-svn] r3416 - trunk/scipy/stats Message-ID: <20071005100832.85AD539C098@new.scipy.org> Author: edschofield Date: 2007-10-05 05:08:16 -0500 (Fri, 05 Oct 2007) New Revision: 3416 Modified: trunk/scipy/stats/kde.py Log: Use pre-computed normalization factor in stats.kde for efficiency Modified: trunk/scipy/stats/kde.py =================================================================== --- trunk/scipy/stats/kde.py 2007-10-04 21:59:25 UTC (rev 3415) +++ trunk/scipy/stats/kde.py 2007-10-05 10:08:16 UTC (rev 3416) @@ -131,8 +131,7 @@ energy = sum(diff*tdiff,axis=0)/2.0 result[i] = sum(exp(-energy),axis=0) - det_cov = linalg.det(2*pi*self.covariance) - result /= sqrt(det_cov)*self.n + result /= self._norm_factor return result From scipy-svn at scipy.org Fri Oct 5 18:40:46 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 5 Oct 2007 17:40:46 -0500 (CDT) Subject: [Scipy-svn] r3417 - trunk/scipy/io Message-ID: <20071005224046.3B70639C077@new.scipy.org> Author: chris.burns Date: 2007-10-05 17:40:42 -0500 (Fri, 05 Oct 2007) New Revision: 3417 Modified: trunk/scipy/io/datasource.py Log: Update documentation and api. Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-05 10:08:16 UTC (rev 3416) +++ trunk/scipy/io/datasource.py 2007-10-05 22:40:42 UTC (rev 3417) @@ -15,6 +15,22 @@ # TODO: replace with newer tuple-based path module from scipy.io.path import path +import warnings + +# datasource has been used for a while in the NiPy project for analyzing +# large fmri imaging files hosted over a network. Data would be fetched +# via URLs, cached locally and analyzed. Under these conditions the code +# worked well, however it needs to be documented, tested and reviewed +# before being fully exposed to SciPy. We hope to do this before the +# 0.7 release. +_api_warning = "The datasource API will be changing frequently before \ +the 0.7 release as the code is ported from the NiPy project to SciPy. \ +Some of the current public interface may become private during the port! \ +Use this module minimally, if at all, until it is stabilized." + +warnings.warn(_api_warning) + +# TODO: .zip support zipexts = (".gz",".bz2") file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file} @@ -46,7 +62,7 @@ *Returns*: - path + path_obj Path object of the unzipped file. """ @@ -81,12 +97,11 @@ return False def splitzipext(filename): - """Return a tuple containing the filename and the zip extension separated. + """Split the filename into a path object and a zip extension. - If the filename does not have a zip extension then: - base -> filename - zip_ext -> None - + If the filename does not have a zip extension the zip_ext in the + return will be None. + *Parameters*: filename : {string} @@ -95,7 +110,7 @@ *Returns*: base, zip_ext : {tuple} - Tuple containing the base file... + Tuple containing a path object to the file and the zip extension. """ @@ -104,12 +119,11 @@ else: return filename, None - - def isurl(pathstr): """Test whether a given string can be parsed as a URL. - *Parameters* + *Parameters*: + pathstr : {string} The string to be checked. @@ -123,10 +137,8 @@ return bool(scheme and netloc) def ensuredirs(directory): - """Ensure that the given directory path actually exists. + """Ensure that the given directory path exists. If not, create it. - If the directory does not exist, it is created. - *Parameters*: directory : {path object} @@ -141,10 +153,10 @@ directory.makedirs() class Cache (object): - """A file cache. + """A local file cache for URL datasources. - The path of the cache can be specified or else use ~/.scipy/cache - by default. + The path of the cache can be specified on intialization. The default + path is ~/.scipy/cache """ @@ -160,36 +172,92 @@ ensuredirs(self.path) def tempfile(self, suffix='', prefix=''): - """ Return an temporary file name in the cache""" + """Create and return a temporary file in the cache. + + *Parameters*: + suffix : {''}, optional + + prefix : {''}, optional + + *Returns*: + tmpfile : {string} + String containing the full path to the temporary file. + + *Examples* + + >>> mycache = datasource.Cache() + >>> mytmpfile = mycache.tempfile() + >>> mytmpfile + '/home/guido/.scipy/cache/GUPhDv' + + """ + _tmp, fname = mkstemp(suffix, prefix, self.path) return fname def filepath(self, uri): + """Return a path object to the uri in the cache. + + *Parameters*: + uri : {string} + Filename to use in the returned path object. + + *Returns*: + path_obj + Path object for the given uri. + + *Examples* + + >>> mycache = datasource.Cache() + >>> mycache.filepath('xyzcoords.txt') + path('/home/guido/.scipy/cache/yzcoords.txt') + """ - Return the complete path + filename within the cache. - """ - (_, netloc, upath, _, _, _) = urlparse(uri) + # TODO: Change to non-public? + # TODO: BUG: First character is removed in the returned path. Why? + # It appears the Cache is designed to work with URLs only! + + (_tmp, netloc, upath, _tmp, _tmp, _tmp) = urlparse(uri) return self.path.joinpath(netloc, upath[1:]) def filename(self, uri): - """ - Return the complete path + filename within the cache. + """Return the complete path + filename within the cache. - :Returns: ``string`` + *Parameters*: + uri : {string} + Filename to usein the returned path. + + *Returns*: + filename + + *Examples* + + >>> mycache = datasource.Cache() + >>> mycache.filename('xyzcoords.txt') + '/home/guido/.scipy/cache/yzcoords.txt' + """ + # TODO: Change to non-public? + return str(self.filepath(uri)) def cache(self, uri): - """ - Copy a file into the cache. + """Copy a file into the cache. :Returns: ``None`` """ if self.iscached(uri): return + + print 'cache uri:', uri + upath = self.filepath(uri) + + print 'upath:', upath + ensuredirs(upath.dirname()) try: + print 'uri:', uri openedurl = urlopen(uri) except: raise IOError("url not found: "+str(uri)) @@ -225,7 +293,8 @@ class DataSource (object): """A generic data source class. - Data could be from a file, URL, cached file. + Data sets could be from a file, a URL, or a cached file. They may also + be compressed or uncompressed. TODO: Improve DataSource docstring From scipy-svn at scipy.org Fri Oct 5 19:49:57 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 5 Oct 2007 18:49:57 -0500 (CDT) Subject: [Scipy-svn] r3418 - trunk/scipy/io/nifti Message-ID: <20071005234957.6B506C7C09C@new.scipy.org> Author: chris.burns Date: 2007-10-05 18:49:54 -0500 (Fri, 05 Oct 2007) New Revision: 3418 Modified: trunk/scipy/io/nifti/README Log: Updating documentation to reST format. Modified: trunk/scipy/io/nifti/README =================================================================== --- trunk/scipy/io/nifti/README 2007-10-05 22:40:42 UTC (rev 3417) +++ trunk/scipy/io/nifti/README 2007-10-05 23:49:54 UTC (rev 3418) @@ -10,69 +10,139 @@ NIfTI +++++ -NIfTI is a new Analyze-style data format, proposed by the NIfTI Data Format Working Group as a "short-term measure to facilitate inter-operation of functional MRI data analysis software packages". +NIfTI is a new Analyze-style data format, proposed by the NIfTI Data Format +Working Group as a "short-term measure to facilitate inter-operation of +functional MRI data analysis software packages". -Meanwhile a number of toolkits are NIfTI-aware (e.g. FSL, AFNI, SPM, Freesurfer and Brainvoyager). Additionally, dicomnifti allows the direct conversion from DICOM images into the NIfTI format. +Meanwhile a number of toolkits are NIfTI-aware (e.g. FSL, AFNI, SPM, +Freesurfer and to a certain degree also Brainvoyager). Additionally, +dicomnifti allows the direct conversion from DICOM images into the NIfTI format. -With libniftiio there is a reference implementation of a C library to read, write and manipulate NIfTI images. The library source code is put into the public domain and a corresponding project is hosted at SourceForge. +With libniftiio there is a reference implementation of a C library to read, +write and manipulate NIfTI images. The library source code is put into the +public domain and a corresponding project is hosted at SourceForge. -In addition to the C library, there is also an IO library written in Java and Matlab functions to make use of NIfTI files from within Matlab. +In addition to the C library, there is also an IO library written in Java and +Matlab functions to make use of NIfTI files from within Matlab. Python ++++++ -Unfortunately, it is not that trivial to read NIfTI images with Python. This is particularly sad, because there is a large number of easy-to-use, high-quality libraries for signal processing available for Python (e.g. SciPy). +Unfortunately, it is not that trivial to read NIfTI images with Python. +This is particularly sad, because there is a large number of easy-to-use, +high-quality libraries for signal processing available for Python (e.g. SciPy). -Moreover Python has bindings to almost any important language/program in the fields of maths, statistics and/or engineering. If you want to use R to calculate some stats in a Python script, simply use RPy and pass any data to R. If you don't care about R, but Matlab is your one and only friend, there are at least two different Python modules to control Matlab from within Python scripts. Python is the glue between all those helpers and the Python user is able to combine as many tools as necessary to solve a given problem -- the easiest way. +Moreover Python has bindings to almost any important language/program in the +fields of maths, statistics and/or engineering. If you want to use R to +calculate some stats in a Python script, simply use RPy and pass any data +to R. If you don't care about R, but Matlab is your one and only friend, +there are at least two different Python modules to control Matlab from within +Python scripts. Python is the glue between all those helpers and the Python +user is able to combine as many tools as necessary to solve a given problem +-- the easiest way. PyNIfTI +++++++ -PyNIfTI aims to provide easy access to NIfTI images from within Python. It uses SWIG-generated wrappers for the NIfTI reference library and provides the NiftiImage class for Python-style access to the image data. +PyNIfTI aims to provide easy access to NIfTI images from within Python. +It uses SWIG-generated wrappers for the NIfTI reference library and +provides the NiftiImage class for Python-style access to the image data. -While PyNIfTI is not yet complete, it already provides access to the most important features of the NIfTI-1 data format and libniftiio capabilities. The following features are currently implemented: +While PyNIfTI is not yet complete (i.e. doesn't support everything the +C library can do), it already provides access to the most important features +of the NIfTI-1 data format and libniftiio capabilities. The following +features are currently implemented: - * PyNIfTI can read and write any file format supported by libniftiio. This includes NIfTI (single and pairs) as well as ANALYZE files. - * PyNIfTI provides fast and convenient access to the image data via NumPy arrays. This should enable users to process image data with most (if not all) numerical routines available for Python. The NumPy array automatically uses a datatype corresponding to the NIfTI image data -- no unnecessary upcasting is performed. - * PyNIfTI provides full read and write access to the NIfTI header data. Header information can be exported to a Python dictionary and can also be updated by using information from a dictionary. - * Instead of accessing NIfTI data from files, PyNIfTI is able to create NIfTI images from NumPy arrays. The appropriate NIfTI header information is determined from the array properties. Additional header information can be optionally specified -- making it easy to clone NIfTI images if necessary, but with minor modifications. - * Most properties of NIfTI images are accessible via attributes and/or accessor functions of the NiftiImage. Inter-dependent properties are automatically updated if necessary (e.g. modifying the Q-Form matrix also updates the pixdim properties and quaternion representation). - * All properties are accessible via Python-style datatypes: A 4x4 matrix is an array not 16 individual numbers. - * PyNIfTI should be resonably fast. Image data will only be loaded into the memory if necessary. Simply opening a NIfTI file to access some header data is performed with virtually no delay independent of the size of the image. Unless image resizing or datatype conversion must be performed the image data can be shared by the NIfTI image and accessing NumPy arrays, and therefore memory won't be wasted memory with redundant copies of the image data. However, one should be careful to make a copy of the image data if you intend to resize and cast the image data (see the docstring of the NiftiImage.asarray() method). + * PyNIfTI can read and write any file format supported by libniftiio. + This includes NIfTI (single and pairs) as well as ANALYZE files. + * PyNIfTI provides fast and convenient access to the image data via NumPy + arrays. This should enable users to process image data with most + (if not all) numerical routines available for Python. The NumPy array + automatically uses a datatype corresponding to the NIfTI image data + -- no unnecessary upcasting is performed. + * PyNIfTI provides full read and write access to the NIfTI header data. + Header information can be exported to a Python dictionary and can also + be updated by using information from a dictionary. + * Instead of accessing NIfTI data from files, PyNIfTI is able to create + NIfTI images from NumPy arrays. The appropriate NIfTI header information + is determined from the array properties. Additional header information + can be optionally specified -- making it easy to clone NIfTI images if + necessary, but with minor modifications. + * Most properties of NIfTI images are accessible via attributes and/or + accessor functions of the NiftiImage. Inter-dependent properties are + automatically updated if necessary (e.g. modifying the Q-Form matrix also + updates the pixdim properties and quaternion representation). + * All properties are accessible via Python-style datatypes: A 4x4 matrix + is an array not 16 individual numbers. + * PyNIfTI should be resonably fast. Image data will only be loaded into + the memory if necessary. Simply opening a NIfTI file to access some header + data is performed with virtually no delay independent of the size of the + image. Unless image resizing or datatype conversion must be performed the + image data can be shared by the NIfTI image and accessing NumPy arrays, + and therefore memory won't be wasted memory with redundant copies of the + image data. However, one should be careful to make a copy of the image data + if you intend to resize and cast the image data (see the docstring of + the NiftiImage.asarray() method). Scripts +++++++ -Some functions provided by PyNIfTI also might be useful outside the Python environment. Therefore I plan to add some command line scripts to the package. -Currently there is only one: pynifti_pst (pst: peristimulus timecourse). Using this script one can compute the signal timecourse for a certain condition for all voxels in a volume at once. This might be useful for exploring a dataset and accompanies similar tools like FSL's tsplot. +Some functions provided by PyNIfTI also might be useful outside the Python +environment. Therefore I plan to add some command line scripts to the package. +Currently there is only one: pynifti_pst (pst: peristimulus timecourse). +Using this script one can compute the signal timecourse for a certain +condition for all voxels in a volume at once. This might be useful for +exploring a dataset and accompanies similar tools like FSL's tsplot. -The output of pynifti_pst can be loaded into FSLView to simultaneously look at statistics and signal timecourses. Please see the corresponding example below. +The output of pynifti_pst can be loaded into FSLView to simultaneously look at +statistics and signal timecourses. Please see the corresponding example below. Known issues aka bugs +++++++++++++++++++++ - * PyNIfTI currently ignores the origin field of ANALYZE files - it is neither read nor written. A possible workaround is to convert ANALYZE files into the NIfTI format using FSL's avwchfiletype. + * PyNIfTI currently ignores the origin field of ANALYZE files - it is + neither read nor written. A possible workaround is to convert ANALYZE + files into the NIfTI format using FSL's avwchfiletype. 2. License ---------- -PyNIfTI is written by Michael Hanke as free software (both beer and speech) and licensed under the GNU Lesser General Public License. +PyNIfTI is written by Michael Hanke as free software (both beer and speech) +and licensed under the MIT License. 3. Download ----------- -As PyNIfTI is still pretty young, a number of significant improvements/modifications are very likely to happen in the near future. If you discover any bugs or you are missing some features, please be sure to check the SVN repository (read below) if your problem is already solved. -Source code +As PyNIfTI is still pretty young, a number of significant +improvements/modifications are very likely to happen in the near future. +If you discover any bugs or you are missing some features, please be sure to +check the SVN repository (read below) to seefif your problem is already solved. -Since June 2007 PyNIfTI is part of the niftilibs family. The PyNIfTI source code can be obtained from the Sourceforge project site. +Source Code +----------- -Source code is also available from the SVN repository of the Debian Experimental Psychology Project. To checkout the latest version use this command: +Since June 2007 PyNIfTI is part of the `niftilibs family +`_. +The PyNIfTI source code can be obtained from the `Sourceforge project site +`_. -svn co svn://svn.debian.org/svn/pkg-exppsy/pynifti/trunk pynifti-latest Binary packages +--------------- -Binary packages for some Debian and (K)Ubuntu versions are available. Please visit this page to read about how you have to setup your system to retrieve the PyNIfTI package via your package manager and stay in sync with future releases. +GNU/Linux +--------- +PyNIfTI is available in recent versions of the Debian (since lenny) and +Ubuntu (since gutsy in universe) distributions. The name of the binary package +is *python-nifti* in both cases: + + * `PyNIfTI versions in Debian `_. + * `PyNIfTI versions in Ubuntu `_. + + +Binary packages for some Debian and (K)Ubuntu versions are also available. +Please visit this page to read about how you have to setup your system to retrieve the PyNIfTI package via your package manager and stay in sync with future releases. + 4. Installation --------------- From scipy-svn at scipy.org Fri Oct 5 20:37:41 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 5 Oct 2007 19:37:41 -0500 (CDT) Subject: [Scipy-svn] r3419 - trunk/scipy/sparse Message-ID: <20071006003741.F2D0339C0F8@new.scipy.org> Author: wnbell Date: 2007-10-05 19:37:39 -0500 (Fri, 05 Oct 2007) New Revision: 3419 Modified: trunk/scipy/sparse/sparse.py Log: added issparse_lil to all Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-05 23:49:54 UTC (rev 3418) +++ trunk/scipy/sparse/sparse.py 2007-10-06 00:37:39 UTC (rev 3419) @@ -9,7 +9,8 @@ 'lil_matrix','dok_matrix', 'spdiags','speye','spidentity','extract_diagonal', 'isspmatrix','issparse','isspmatrix_csc','isspmatrix_csr', - 'isspmatrix_lil','isspmatrix_dok', 'lil_eye', 'lil_diags' ] + 'isspmatrix_lil','isspmatrix_dok', 'isspmatrix_coo', + 'lil_eye', 'lil_diags' ] import warnings From scipy-svn at scipy.org Sat Oct 6 11:48:01 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sat, 6 Oct 2007 10:48:01 -0500 (CDT) Subject: [Scipy-svn] r3420 - trunk/scipy/optimize Message-ID: <20071006154801.2774E39C037@new.scipy.org> Author: dmitrey.kroshko Date: 2007-10-06 10:47:55 -0500 (Sat, 06 Oct 2007) New Revision: 3420 Modified: trunk/scipy/optimize/minpack.py Log: update in docstring about calling leastsq from scikits.openopt Modified: trunk/scipy/optimize/minpack.py =================================================================== --- trunk/scipy/optimize/minpack.py 2007-10-06 00:37:39 UTC (rev 3419) +++ trunk/scipy/optimize/minpack.py 2007-10-06 15:47:55 UTC (rev 3420) @@ -242,6 +242,8 @@ See also: + scikits.openopt, which offers a unified syntax to call this and other solvers + fmin, fmin_powell, fmin_cg, fmin_bfgs, fmin_ncg -- multivariate local optimizers leastsq -- nonlinear least squares minimizer From scipy-svn at scipy.org Sun Oct 7 02:08:07 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 7 Oct 2007 01:08:07 -0500 (CDT) Subject: [Scipy-svn] r3421 - trunk/scipy/io Message-ID: <20071007060807.88D26C7C007@new.scipy.org> Author: brian.hawthorne Date: 2007-10-07 01:08:03 -0500 (Sun, 07 Oct 2007) New Revision: 3421 Modified: trunk/scipy/io/datasource.py Log: bugfix datasource handling of leading slash Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-06 15:47:55 UTC (rev 3420) +++ trunk/scipy/io/datasource.py 2007-10-07 06:08:03 UTC (rev 3421) @@ -17,14 +17,14 @@ import warnings -# datasource has been used for a while in the NiPy project for analyzing +# datasource has been used for a while in the NIPY project for analyzing # large fmri imaging files hosted over a network. Data would be fetched # via URLs, cached locally and analyzed. Under these conditions the code # worked well, however it needs to be documented, tested and reviewed # before being fully exposed to SciPy. We hope to do this before the # 0.7 release. _api_warning = "The datasource API will be changing frequently before \ -the 0.7 release as the code is ported from the NiPy project to SciPy. \ +the 0.7 release as the code is ported from the NIPY project to SciPy. \ Some of the current public interface may become private during the port! \ Use this module minimally, if at all, until it is stabilized." @@ -210,15 +210,14 @@ >>> mycache = datasource.Cache() >>> mycache.filepath('xyzcoords.txt') - path('/home/guido/.scipy/cache/yzcoords.txt') + path('/home/guido/.scipy/cache/xyzcoords.txt') """ # TODO: Change to non-public? - # TODO: BUG: First character is removed in the returned path. Why? + # It appears the Cache is designed to work with URLs only! - (_tmp, netloc, upath, _tmp, _tmp, _tmp) = urlparse(uri) - return self.path.joinpath(netloc, upath[1:]) + return self.path.joinpath(netloc, upath.strip('/')) def filename(self, uri): """Return the complete path + filename within the cache. @@ -364,6 +363,7 @@ """ #"""DataSource with an implied root.""" + def __init__(self, baseurl, cachepath=None): DataSource.__init__(self, cachepath=cachepath) self._baseurl = baseurl From scipy-svn at scipy.org Sun Oct 7 02:10:09 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 7 Oct 2007 01:10:09 -0500 (CDT) Subject: [Scipy-svn] r3422 - trunk/scipy/io/nifti Message-ID: <20071007061009.6A6DF39C012@new.scipy.org> Author: brian.hawthorne Date: 2007-10-07 01:10:07 -0500 (Sun, 07 Oct 2007) New Revision: 3422 Removed: trunk/scipy/io/nifti/nifticlib.py Log: generated file Deleted: trunk/scipy/io/nifti/nifticlib.py =================================================================== From scipy-svn at scipy.org Mon Oct 8 05:57:18 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 8 Oct 2007 04:57:18 -0500 (CDT) Subject: [Scipy-svn] r3423 - in trunk/scipy/sparse: . tests Message-ID: <20071008095718.E7B5539C080@new.scipy.org> Author: rc Date: 2007-10-08 04:57:10 -0500 (Mon, 08 Oct 2007) New Revision: 3423 Modified: trunk/scipy/sparse/sparse.py trunk/scipy/sparse/tests/test_sparse.py Log: issequence() accepts numpy 1D arrays -> those can be used as lil matrix indices. Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-07 06:10:07 UTC (rev 3422) +++ trunk/scipy/sparse/sparse.py 2007-10-08 09:57:10 UTC (rev 3423) @@ -2926,4 +2926,5 @@ return out def issequence(t): - return isinstance(t, (list, tuple)) + return isinstance(t, (list, tuple))\ + or (isinstance(t, ndarray) and (t.ndim == 1)) Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-07 06:10:07 UTC (rev 3422) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-08 09:57:10 UTC (rev 3423) @@ -940,6 +940,19 @@ B[:2,:2] = csc_matrix(array(block)) assert_array_equal(B.todense()[:2,:2],block) + def check_lil_sequence_assignement(self): + A = lil_matrix((4,3)) + B = lil_eye((3,4)) + + i0 = [0,1,2] + i1 = (0,1,2) + i2 = array( i0 ) + + A[0,i0] = B[i0,0] + A[1,i1] = B[i1,1] + A[2,i2] = B[i2,2] + assert_array_equal(A.todense(),B.T.todense()) + def check_lil_iteration(self): row_data = [[1,2,3],[4,5,6]] B = lil_matrix(array(row_data)) From scipy-svn at scipy.org Mon Oct 8 15:10:56 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 8 Oct 2007 14:10:56 -0500 (CDT) Subject: [Scipy-svn] r3424 - trunk/scipy/io/nifti Message-ID: <20071008191056.E523C39C019@new.scipy.org> Author: chris.burns Date: 2007-10-08 14:10:54 -0500 (Mon, 08 Oct 2007) New Revision: 3424 Modified: trunk/scipy/io/nifti/README Log: Update documentation. Modified: trunk/scipy/io/nifti/README =================================================================== --- trunk/scipy/io/nifti/README 2007-10-08 09:57:10 UTC (rev 3423) +++ trunk/scipy/io/nifti/README 2007-10-08 19:10:54 UTC (rev 3424) @@ -10,17 +10,29 @@ NIfTI +++++ -NIfTI is a new Analyze-style data format, proposed by the NIfTI Data Format -Working Group as a "short-term measure to facilitate inter-operation of -functional MRI data analysis software packages". +.. _NIfTI: http://nifti.nimh.nih.gov/ +.. _SourceForge: http://sourceforge.net/projects/niftilib +.. _SWIG: http://www.swig.org/ +.. _NumPy: http://numpy.scipy.org/ +.. _Matplotlib: http://matplotlib.sourceforge.net/ +.. _R: http://www.r-project.org/ +.. _RPy: http://rpy.sourceforge.net/ +NIfTI_ is a new Analyze-style data format, proposed by the +`NIfTI Data Format Working Group +`_ +as a *"short-term measure to facilitate inter-operation of +functional MRI data analysis software packages".* + Meanwhile a number of toolkits are NIfTI-aware (e.g. FSL, AFNI, SPM, Freesurfer and to a certain degree also Brainvoyager). Additionally, -dicomnifti allows the direct conversion from DICOM images into the NIfTI format. +`dicomnifti `_ +allows the direct conversion from DICOM images into the NIfTI format. -With libniftiio there is a reference implementation of a C library to read, +With `libniftiio `_ +there is a reference implementation of a C library to read, write and manipulate NIfTI images. The library source code is put into the -public domain and a corresponding project is hosted at SourceForge. +public domain and a corresponding project is hosted at SourceForge_. In addition to the C library, there is also an IO library written in Java and Matlab functions to make use of NIfTI files from within Matlab. @@ -33,19 +45,19 @@ high-quality libraries for signal processing available for Python (e.g. SciPy). Moreover Python has bindings to almost any important language/program in the -fields of maths, statistics and/or engineering. If you want to use R to -calculate some stats in a Python script, simply use RPy and pass any data -to R. If you don't care about R, but Matlab is your one and only friend, -there are at least two different Python modules to control Matlab from within -Python scripts. Python is the glue between all those helpers and the Python -user is able to combine as many tools as necessary to solve a given problem --- the easiest way. +fields of maths, statistics and/or engineering. If you want to use R_ to +calculate some stats in a Python script, simply use RPy_ +and pass any data to R_. If you don't care about R_, but Matlab is your one +and only friend, there are at least two different Python modules to control +Matlab from within Python scripts. Python is the glue between all those helpers +and the Python user is able to combine as many tools as necessary to solve a +given problem -- the easiest way. PyNIfTI +++++++ PyNIfTI aims to provide easy access to NIfTI images from within Python. -It uses SWIG-generated wrappers for the NIfTI reference library and +It uses SWIG_ -generated wrappers for the NIfTI reference library and provides the NiftiImage class for Python-style access to the image data. While PyNIfTI is not yet complete (i.e. doesn't support everything the @@ -55,7 +67,7 @@ * PyNIfTI can read and write any file format supported by libniftiio. This includes NIfTI (single and pairs) as well as ANALYZE files. - * PyNIfTI provides fast and convenient access to the image data via NumPy + * PyNIfTI provides fast and convenient access to the image data via NumPy_ arrays. This should enable users to process image data with most (if not all) numerical routines available for Python. The NumPy array automatically uses a datatype corresponding to the NIfTI image data @@ -107,8 +119,9 @@ 2. License ---------- -PyNIfTI is written by Michael Hanke as free software (both beer and speech) -and licensed under the MIT License. +PyNIfTI is written by `Michael Hanke `_ +as free software (both beer and speech) and licensed under the `MIT License +`_. 3. Download ----------- @@ -119,18 +132,17 @@ check the SVN repository (read below) to seefif your problem is already solved. Source Code ------------ ++++++++++++ Since June 2007 PyNIfTI is part of the `niftilibs family `_. -The PyNIfTI source code can be obtained from the `Sourceforge project site -`_. +The PyNIfTI source code can be obtained from the SourceForge_ project site. Binary packages ---------------- ++++++++++++++++ GNU/Linux ---------- ++++++++++ PyNIfTI is available in recent versions of the Debian (since lenny) and Ubuntu (since gutsy in universe) distributions. The name of the binary package @@ -141,45 +153,56 @@ Binary packages for some Debian and (K)Ubuntu versions are also available. -Please visit this page to read about how you have to setup your system to retrieve the PyNIfTI package via your package manager and stay in sync with future releases. +Please visit this page to read about how you have to setup your system to +retrieve the PyNIfTI package via your package manager and stay in sync with +future releases. -4. Installation ---------------- +Windows ++++++++ -Binary packages -+++++++++++++++ +A binary installer for a recent Python version is available from the +SourceForge_ project site. -If you have configured your system as described on this page all you have to do to install PyNIfTI is this:: +Macintosh ++++++++++ +Unfortunately, no binary packages are available. I have no access to such a +machine at the moment. But it is possible to build PyNIfTI from source on +Mac OS X (see below for more information). - apt-get update - apt-get install python-nifti -This should pull all necessary dependencies. +4. Installation +--------------- -Compile from source -+++++++++++++++++++ +Compile from source: General instructions ++++++++++++++++++++++++++++++++++++++++++ PyNIfTI needs a few things to build and run properly: - * Python 2.3 or greater - * NumPy - * SWIG - * NIfTI C libraries + * `Python `_ 2.3 or greater + * NumPy_ + * SWIG_ + * `NIfTI C libraries `_ -Make sure that the compiled nifticlibs and the corresponding headers are available to your compiler. If they are located in a custom directory, you might have to specify --include-dirs and --library-dirs options to the build command below. +Make sure that the compiled nifticlibs and the corresponding headers are +available to your compiler. If they are located in a custom directory, you +might have to specify --include-dirs and --library-dirs options to the +build command below. -Once you have downloaded the sources, extract the tarball and enter the root directory of the extracted sources. A simple:: +Once you have downloaded the sources, extract the tarball and enter the +root directory of the extracted sources. A simple:: python setup.py build_ext -should build the SWIG wrappers. If this has been done successfully, all you need to do is install the modules by invoking:: +should build the SWIG wrappers. If this has been done successfully, all +you need to do is install the modules by invoking:: sudo python setup.py install If sudo is not configured (or even installed) you might have to use su instead. -Now fire up Python and try importing the module to see if everything is fine. It should look similar to this:: +Now fire up Python and try importing the module to see if everything is fine. +It should look similar to this:: Python 2.4.4 (#2, Oct 20 2006, 00:23:25) [GCC 4.1.2 20061015 (prerelease) (Debian 4.1.1-16.1)] on linux2 @@ -187,33 +210,75 @@ >>> import nifti >>> +Windows ++++++++ + +It should be pretty straightforward to compile PyNIfTI for win32. The most +convenient way seems to be using the `Dev-Cpp IDE +`_ and the DevPack of the nifticlibs. +Have a look into the toplevel Makefile of the PyNIfTI source distribution for +some hints. + MacOS X and MacPython +++++++++++++++++++++ -When you are comiling PyNIfTI on MacOS X and want to use it with MacPython, please make sure that the NIfTI C libraries are compiled as fat binaries (compiled for both ppc and i386). Otherwise PyNIfTI extensions will not compile. +When you are comiling PyNIfTI on MacOS X and want to use it with MacPython, +please make sure that the NIfTI C libraries are compiled as fat binaries +(compiled for both ppc and i386). Otherwise PyNIfTI extensions will not compile. -One can achieve this by adding both architectures to the CFLAGS definition in the toplevel Makefile of the NIfTI C library source code. Like this:: +One can achieve this by adding both architectures to the CFLAGS definition +in the toplevel Makefile of the NIfTI C library source code. Like this:: CFLAGS = $(ANSI_FLAGS) -arch ppc -arch i386 +Binary packages ++++++++++++++++ + +GNU/Linux ++++++++++ + +If you have configured your system as described on this page all you have to +do to install PyNIfTI is this:: + + apt-get update + apt-get install python-nifti + +This should pull all necessary dependencies. If it doesn't, it's a bug that +should be reported. + +Windows ++++++++ + +As always, click Next as long as necessary and finally Finish. + Troubleshooting +++++++++++++++ -If you get an error when importing the nifti module in Python complaining about missing symbols your niftiio library contains references to some unresolved symbols. Try adding znzlib and zlib to the linker options the PyNIfTI setup.py, like this:: +If you get an error when importing the nifti module in Python complaining +about missing symbols your niftiio library contains references to some +unresolved symbols. Try adding znzlib and zlib to the linker options the +PyNIfTI setup.py, like this:: libraries = [ 'niftiio', 'znz', 'z' ], 5. Things to know ----------------- -When accessing NIfTI image data through NumPy arrays the order of the dimensions is reversed. If the x, y, z, t dimensions of a NIfTI image are 64, 64, 32, 456 (as for example reported by nifti_tool), the shape of the NumPy array (e.g. as returned by NiftiImage.asarray()) will be: 456, 32, 64, 64. +When accessing NIfTI image data through NumPy arrays the order of the +dimensions is reversed. If the x, y, z, t dimensions of a NIfTI image are +64, 64, 32, 456 (as for example reported by nifti_tool), the shape of the +NumPy array (e.g. as returned by NiftiImage.asarray()) will be: 456, 32, 64, 64. -This is done to be able to slice the data array much easier in the most common cases. For example, if you are interested in a certain volume of a timeseries it is much easier to write data[2] instead of data[:,:,:,2], right?. +This is done to be able to slice the data array much easier in the most common +cases. For example, if you are interested in a certain volume of a timeseries +it is much easier to write data[2] instead of data[:,:,:,2], right?. 6. Examples ----------- -The next sections contains some examples showing ways to use PyNIfTI to read and write imaging data from within Python to be able to process it with some random Python library. +The next sections contains some examples showing ways to use PyNIfTI to read +and write imaging data from within Python to be able to process it with some +random Python library. All examples assume that you have imported the PyNIfTI module by invoking:: @@ -222,7 +287,8 @@ a) Fileformat conversion ++++++++++++++++++++++++ -Open the MNI standard space template that is shipped with FSL. No filename extension is necessary as libniftiio determines it automatically:: +Open the MNI standard space template that is shipped with FSL. No filename +extension is necessary as libniftiio determines it automatically:: nim = NiftiImage('avg152T1_brain') @@ -230,41 +296,55 @@ print nim.filename -yields 'avg152T1_brain.img'. This indicates an ANALYZE image. If you want to save this image as a single gzipped NIfTI file simply do:: +yields 'avg152T1_brain.img'. This indicates an ANALYZE image. If you want to +save this image as a single gzipped NIfTI file simply do:: nim.save('mni.nii.gz') -The filetype is determined from the filename. If you want to save to gzipped ANALYZE file pairs instead the following would be an alternative to calling the save() with a new filename:: +The filetype is determined from the filename. If you want to save to gzipped +ANALYZE file pairs instead the following would be an alternative to calling +the save() with a new filename:: nim.filename = 'mni_analyze.img.gz' nim.save() -Please see the docstring of the NiftiImage.setFilename() method to learn how the filetypes are determined from the filenames. +Please see the docstring of the NiftiImage.setFilename() method to learn how +the filetypes are determined from the filenames. b) NIfTI files from array data ++++++++++++++++++++++++++++++ -The next code snipped demonstrates how to create a 4d NIfTI image containing gaussian noise. First we need to import the NumPy module:: +The next code snipped demonstrates how to create a 4d NIfTI image containing +gaussian noise. First we need to import the NumPy module:: import numpy -Now generate the noise dataset. Let's generate noise for 100 volumes with 16 slices and a 32x32 inplane matrix:: +Now generate the noise dataset. Let's generate noise for 100 volumes with 16 +slices and a 32x32 inplane matrix:: noise = numpy.random.randn(100,16,32,32) Please notice the order in which the dimensions are specified: (t, z, y, x). -The datatype of the array will most likely be float64 -- which can be verified by invoking noise.dtype. +The datatype of the array will most likely be float64 -- which can be verified +by invoking noise.dtype. -Converting this dataset into a NIfTI image is done by invoking the NiftiImage constructor with the noise dataset as argument:: +Converting this dataset into a NIfTI image is done by invoking the NiftiImage +constructor with the noise dataset as argument:: nim = NiftiImage(noise) -The relevant header information is extracted from the NumPy array. If you query the header information about the dimensionality of the image, it returns the desired values:: +The relevant header information is extracted from the NumPy array. If you query +the header information about the dimensionality of the image, it returns the +desired values:: print nim.header['dim'] # yields: [4, 32, 32, 16, 100, 0, 0, 0] -First value shows the number of dimensions in the datset: 4 (good, that's what we wanted). The following numbers are dataset size on the x, y, z, t, u, v, w axis (NIfTI files can handle up to 7 dimensions). Please notice, that the order of dimensions is now 'correct': We have 32x32 inplane resolution, 16 slices in z direction and 100 volumes. +First value shows the number of dimensions in the datset: 4 (good, that's what +we wanted). The following numbers are dataset size on the x, y, z, t, u, v, w +axis (NIfTI files can handle up to 7 dimensions). Please notice, that the order +of dimensions is now 'correct': We have 32x32 inplane resolution, 16 slices +in z direction and 100 volumes. Also the datatype was set appropriately. The expression:: @@ -279,48 +359,73 @@ c) Select ROIs ++++++++++++++ -Suppose you want to have the first ten volumes of the noise dataset we have just created in a separate file. First open the file (can be skipped if it is still open):: +Suppose you want to have the first ten volumes of the noise dataset we have +just created in a separate file. First open the file (can be skipped if it is +still open):: nim = NiftiImage('noise.nii.gz') -Now select the first ten volumes and store them to another file, while preserving as much header information as possible:: +Now select the first ten volumes and store them to another file, while +preserving as much header information as possible:: nim2 = NiftiImage(nim.data[:10], nim.header) nim2.save('part.hdr.gz') -The NiftiImage constructor takes a dictionary with header information as an optional argument. Settings that are not determined by the array (e.g. size, datatype) are copied from the dictionary and stored to the new NIfTI image. +The NiftiImage constructor takes a dictionary with header information as an +optional argument. Settings that are not determined by the array (e.g. size, +datatype) are copied from the dictionary and stored to the new NIfTI image. d) Linear detrending of timeseries ++++++++++++++++++++++++++++++++++ -Let's load another 4d NIfTI file and perform a linear detrending, by fitting a straight line to the timeseries of each voxel and substract that fit from the data. Although this might sound complicated at first, thanks to the excellent SciPy module it is just a few lines of code:: +*Note: SciPy module is required for this example.* +Let's load another 4d NIfTI file and perform a linear detrending, by fitting +a straight line to the timeseries of each voxel and substract that fit from +the data. Although this might sound complicated at first, thanks to the +excellent SciPy module it is just a few lines of code:: + nim = NiftiImage('timeseries.nii') -Depending on the datatype of the input image the detrending process might change the datatype from integer to float. As operations that change the (binary) size of the NIfTI image are not supported, we need to make a copy of the data and later create a new NIfTI image:: +Depending on the datatype of the input image the detrending process might +change the datatype from integer to float. As operations that change the +(binary) size of the NIfTI image are not supported, we need to make a copy +of the data and later create a new NIfTI image:: data = nim.asarray() -Now detrend the data along the time axis. Remember that the array has the time axis as its first dimension (in contrast to the NIfTI file where it is the 4th):: +Now detrend the data along the time axis. Remember that the array has the time +axis as its first dimension (in contrast to the NIfTI file where it is the +4th):: from scipy import signal data_detrended = signal.detrend( data, axis=0 ) -Finally, create a new NIfTI image using header information from the original source image:: +Finally, create a new NIfTI image using header information from the original +source image:: nim_detrended = NiftiImage( data_detrended, nim.header) -e) Make a quick plot of a voxels timeseries (Gnuplot module is required) -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +e) Make a quick plot of a voxels timeseries ++++++++++++++++++++++++++++++++++++++++++++ -Plotting is essential to get a 'feeling' for the data. The python interface to Gnuplot makes it really easy to plot something (e.g. when running Python interactively via IPython). Please note, that there are many other possibilities for plotting. Some examples are: using R via RPy or Matlab-style plotting via matplotlib. +*Note: Gnuplot module is required.* -However, using Gnuplot is really easy. First import the Gnuplot module and create the interface object:: +Plotting is essential to get a "feeling" for the data. The `python interface +`_ to `Gnuplot `_ +makes it really easy to plot something (e.g. when running Python interactively +via `IPython `_). Please note, that there are many +other possibilities for plotting. Some examples are: using +R_ via RPy_ or Matlab-style plotting via Matplotlib_ . +However, using Gnuplot is really easy. First import the Gnuplot module and +create the interface object:: + from Gnuplot import Gnuplot gp = Gnuplot() -We want the timeseries as a line plot and not just the datapoints, so let's talk with Gnuplot:: +We want the timeseries as a line plot and not just the datapoints, so let's +talk with Gnuplot:: gp('set data style lines') @@ -332,13 +437,24 @@ gp.plot(nim.data[:,12,30,20]) -A Gnuplot window showing the timeseries should popup now (screenshot). Please read the Gnuplot Manual to learn what it can do -- and it can do a lot more than just simple line plots (have a look at this page if you are interested). -f) Show a slice of a 3d volume (Matplotlib module is required) +A Gnuplot window showing the timeseries should popup now (`screenshot +`_). +Please read the Gnuplot Manual to learn what it can do -- and it can do a lot +more than just simple line plots (have a look at `this +`_ page if you are +interested). -This example demonstrates howto use the Matlab-style plotting of Matplotlib to view a slice from a 3d volume. +f) Show a slice of a 3d volume +++++++++++++++++++++++++++++++ -This time I assume that a 3d nifti file is already opened and available in the nim3d object. At first we need to load the necessary Python module:: +*Note: Matplotlib module is required.* +This example demonstrates how to use the Matlab-style plotting of Matplotlib_ +to view a slice from a 3d volume. + +This time I assume that a 3d nifti file is already opened and available in the +nim3d object. At first we need to load the necessary Python module:: + from pylab import * If everything went fine, we can now view a slice (x,y):: @@ -346,35 +462,64 @@ imshow(nim3d.data[200], interpolation='nearest', cmap=cm.gray) show() -It is necessary to call the show() function one time after importing pylab to actually see the image when running Python interactively (screenshot). +It is necessary to call the show() function one time after importing pylab +to actually see the image when running Python interactively (`screenshot xyslice +`_ +). When you want to have a look at a yz-slice, NumPy array magic comes into play:: imshow(nim3d.data[::-1,:,100], interpolation='nearest', cmap=cm.gray) -The ::-1 notation causes the z-axis to be flipped in the images. This makes a much nicer screenshot, because the used example volume has the z-axis originally oriented upsidedown. +The ::-1 notation causes the z-axis to be flipped in the images. This makes a +much nicer `screenshot yzslice +`_ +, because the used example volume has the z-axis originally oriented upsidedown. -g) Compute and display peristimulus signal timecourse of multiple conditions with pynifti_pst and FSLView -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +g) Peristimulus signal timecourse of multiple conditions +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -Sometimes one wants to look at the signal timecourse of some voxel after a certain stimulation onset. An easy way would be to have some fMRI data viewer that displays a statistical map and one could click on some activated voxel and the peristimulus signal timecourse of some condition in that voxel would be displayed. +Sometimes one wants to look at the signal timecourse of some voxel after a +certain stimulation onset. An easy way would be to have some fMRI data viewer +that displays a statistical map and one could click on some activated voxel and +the peristimulus signal timecourse of some condition in that voxel would be +displayed. -This can easily be done by using pynifti_pst and FSLView. +This can easily be done by using *pynifti_pst* and *FSLView*. -pynifti_pst comes with a manpage that explains all options and arguments. Basically pynifti_pst need a 4d image (e.g. an fMRI timeseries; possibly preprocessed/filtered) and some stimulus onset information. This information can either be given directly on the command line or is read from files. Additionally one can specify onsets as volume numbers or as onset times. +*pynifti_pst* comes with a manpage that explains all options and arguments. +Basically *pynifti_pst* need a 4d image (e.g. an fMRI timeseries; possibly +preprocessed/filtered) and some stimulus onset information. This information +can either be given directly on the command line or is read from files. +Additionally one can specify onsets as volume numbers or as onset times. -pynifti_pst understands the FSL custom EV file format so one can easily use those files as input. +*pynifti_pst* understands the FSL custom EV file format so one can easily use +those files as input. -An example call could look like this: +An example call could look like this:: -pynifti_pst --times --nvols 5 -p uf92.feat/filtered_func_data.nii.gz pst_cond_a.nii.gz uf92.feat/custom_timing_files/ev1.txt uf92.feat/custom_timing_files/ev2.txt + pynifti_pst --times --nvols 5 -p uf92.feat/filtered_func_data.nii.gz + pst_cond_a.nii.gz uf92.feat/custom_timing_files/ev1.txt + uf92.feat/custom_timing_files/ev2.txt -This computes a peristimulus timeseries using the preprocessed fMRI from a FEAT output directory and two custom EV files that both together make up condition A. --times indicates that the EV files list onset times (not volume ids) and --nvols requests the mean peristimulus timecourse for 4 volumes after stimulus onset (5 including onset). -p recodes the peristimulus timeseries into percent signalchange, where the onset is always zero and any following value is the signal change with respect to the onset volume. +This computes a peristimulus timeseries using the preprocessed fMRI from a +FEAT output directory and two custom EV files that both together make up +condition A. --times indicates that the EV files list onset times (not volume +ids) and --nvols requests the mean peristimulus timecourse for 4 volumes after +stimulus onset (5 including onset). -p recodes the peristimulus timeseries into +percent signalchange, where the onset is always zero and any following value +is the signal change with respect to the onset volume. -This call produces a simple 4d NIfTI image that can be loaded into FSLView as any other timeseries. The following call can be used to display an FSL zmap from the above results path on top of some anatomy. Additionally the peristimulus timeseries of two conditions are loaded. This screenshot shows how it could look like. One of the nice features of FSLView is that its timeseries window can remember selected curves, which can be useful to compare signal timecourses from different voxels (blue and green line in the screenshot). +This call produces a simple 4d NIfTI image that can be loaded into FSLView as +any other timeseries. The following call can be used to display an FSL zmap +from the above results path on top of some anatomy. Additionally the +peristimulus timeseries of two conditions are loaded. `This screenshot +`_ +shows how it could look like. One of the nice features of FSLView is that its +timeseries window can remember selected curves, which can be useful to compare +signal timecourses from different voxels (blue and green line in the +screenshot):: -fslview pst_cond_a.nii.gz pst_cond_b.nii.gz uf92_ana.nii.gz uf92.feat/stats/zstat1.nii.gz -b 3,5 + fslview pst_cond_a.nii.gz pst_cond_b.nii.gz uf92_ana.nii.gz + uf92.feat/stats/zstat1.nii.gz -b 3,5 -History - -The full changelog is here. From scipy-svn at scipy.org Mon Oct 8 15:43:51 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 8 Oct 2007 14:43:51 -0500 (CDT) Subject: [Scipy-svn] r3425 - trunk/scipy/io/nifti Message-ID: <20071008194351.839CB39C0D7@new.scipy.org> Author: chris.burns Date: 2007-10-08 14:43:45 -0500 (Mon, 08 Oct 2007) New Revision: 3425 Modified: trunk/scipy/io/nifti/README Log: Update documentation. Modified: trunk/scipy/io/nifti/README =================================================================== --- trunk/scipy/io/nifti/README 2007-10-08 19:10:54 UTC (rev 3424) +++ trunk/scipy/io/nifti/README 2007-10-08 19:43:45 UTC (rev 3425) @@ -14,6 +14,7 @@ .. _SourceForge: http://sourceforge.net/projects/niftilib .. _SWIG: http://www.swig.org/ .. _NumPy: http://numpy.scipy.org/ +.. _SciPy: http://www.scipy.org/ .. _Matplotlib: http://matplotlib.sourceforge.net/ .. _R: http://www.r-project.org/ .. _RPy: http://rpy.sourceforge.net/ @@ -42,7 +43,7 @@ Unfortunately, it is not that trivial to read NIfTI images with Python. This is particularly sad, because there is a large number of easy-to-use, -high-quality libraries for signal processing available for Python (e.g. SciPy). +high-quality libraries for signal processing available for Python (e.g. SciPy_). Moreover Python has bindings to almost any important language/program in the fields of maths, statistics and/or engineering. If you want to use R_ to @@ -58,11 +59,11 @@ PyNIfTI aims to provide easy access to NIfTI images from within Python. It uses SWIG_ -generated wrappers for the NIfTI reference library and -provides the NiftiImage class for Python-style access to the image data. +provides the ``NiftiImage`` class for Python-style access to the image data. While PyNIfTI is not yet complete (i.e. doesn't support everything the C library can do), it already provides access to the most important features -of the NIfTI-1 data format and libniftiio capabilities. The following +of the NIfTI-1 data format and *libniftiio* capabilities. The following features are currently implemented: * PyNIfTI can read and write any file format supported by libniftiio. @@ -81,32 +82,33 @@ can be optionally specified -- making it easy to clone NIfTI images if necessary, but with minor modifications. * Most properties of NIfTI images are accessible via attributes and/or - accessor functions of the NiftiImage. Inter-dependent properties are + accessor functions of the ``NiftiImage.`` Inter-dependent properties are automatically updated if necessary (e.g. modifying the Q-Form matrix also updates the pixdim properties and quaternion representation). * All properties are accessible via Python-style datatypes: A 4x4 matrix is an array not 16 individual numbers. - * PyNIfTI should be resonably fast. Image data will only be loaded into + * PyNIfTI should be reasonably fast. Image data will only be loaded into the memory if necessary. Simply opening a NIfTI file to access some header - data is performed with virtually no delay independent of the size of the - image. Unless image resizing or datatype conversion must be performed the + data is performed with virtually no delay, independent of the size of the + image. Unless image resizing or datatype conversion must be performed, the image data can be shared by the NIfTI image and accessing NumPy arrays, and therefore memory won't be wasted memory with redundant copies of the image data. However, one should be careful to make a copy of the image data if you intend to resize and cast the image data (see the docstring of - the NiftiImage.asarray() method). + the ``NiftiImage.asarray()`` method). Scripts +++++++ Some functions provided by PyNIfTI also might be useful outside the Python environment. Therefore I plan to add some command line scripts to the package. -Currently there is only one: pynifti_pst (pst: peristimulus timecourse). + +Currently there is only one: *pynifti_pst* (pst: peristimulus timecourse). Using this script one can compute the signal timecourse for a certain condition for all voxels in a volume at once. This might be useful for exploring a dataset and accompanies similar tools like FSL's tsplot. -The output of pynifti_pst can be loaded into FSLView to simultaneously look at +The output of *pynifti_pst* can be loaded into FSLView to simultaneously look at statistics and signal timecourses. Please see the corresponding example below. Known issues aka bugs @@ -114,7 +116,7 @@ * PyNIfTI currently ignores the origin field of ANALYZE files - it is neither read nor written. A possible workaround is to convert ANALYZE - files into the NIfTI format using FSL's avwchfiletype. + files into the NIfTI format using FSL's *avwchfiletype.* 2. License ---------- @@ -186,7 +188,7 @@ Make sure that the compiled nifticlibs and the corresponding headers are available to your compiler. If they are located in a custom directory, you -might have to specify --include-dirs and --library-dirs options to the +might have to specify ``--include-dirs`` and ``--library-dirs`` options to the build command below. Once you have downloaded the sources, extract the tarball and enter the @@ -199,7 +201,8 @@ sudo python setup.py install -If sudo is not configured (or even installed) you might have to use su instead. +If sudo is not configured (or even installed) you might have to use ``su`` +instead. Now fire up Python and try importing the module to see if everything is fine. It should look similar to this:: @@ -224,9 +227,10 @@ When you are comiling PyNIfTI on MacOS X and want to use it with MacPython, please make sure that the NIfTI C libraries are compiled as fat binaries -(compiled for both ppc and i386). Otherwise PyNIfTI extensions will not compile. +(compiled for both *ppc* and *i386*). Otherwise PyNIfTI extensions will not +compile. -One can achieve this by adding both architectures to the CFLAGS definition +One can achieve this by adding both architectures to the ``CFLAGS`` definition in the toplevel Makefile of the NIfTI C library source code. Like this:: CFLAGS = $(ANSI_FLAGS) -arch ppc -arch i386 @@ -249,15 +253,15 @@ Windows +++++++ -As always, click Next as long as necessary and finally Finish. +As always, click *Next* as long as necessary and finally *Finish.* Troubleshooting +++++++++++++++ -If you get an error when importing the nifti module in Python complaining +If you get an error when importing the *nifti* module in Python complaining about missing symbols your niftiio library contains references to some -unresolved symbols. Try adding znzlib and zlib to the linker options the -PyNIfTI setup.py, like this:: +unresolved symbols. Try adding *znzlib* and *zlib* to the linker options the +PyNIfTI ``setup.py``, like this:: libraries = [ 'niftiio', 'znz', 'z' ], @@ -266,12 +270,13 @@ When accessing NIfTI image data through NumPy arrays the order of the dimensions is reversed. If the x, y, z, t dimensions of a NIfTI image are -64, 64, 32, 456 (as for example reported by nifti_tool), the shape of the -NumPy array (e.g. as returned by NiftiImage.asarray()) will be: 456, 32, 64, 64. +64, 64, 32, 456 (as for example reported by *nifti_tool*), the shape of the +NumPy array (e.g. as returned by ``NiftiImage.asarray()``) +will be: 456, 32, 64, 64. This is done to be able to slice the data array much easier in the most common cases. For example, if you are interested in a certain volume of a timeseries -it is much easier to write data[2] instead of data[:,:,:,2], right?. +it is much easier to write ``data[2]`` instead of ``data[:,:,:,2]``, right?. 6. Examples ----------- @@ -303,7 +308,7 @@ The filetype is determined from the filename. If you want to save to gzipped ANALYZE file pairs instead the following would be an alternative to calling -the save() with a new filename:: +the ``save()`` with a new filename:: nim.filename = 'mni_analyze.img.gz' nim.save() @@ -326,8 +331,8 @@ Please notice the order in which the dimensions are specified: (t, z, y, x). -The datatype of the array will most likely be float64 -- which can be verified -by invoking noise.dtype. +The datatype of the array will most likely be *float64* -- which can be +verified by invoking ``noise.dtype``. Converting this dataset into a NIfTI image is done by invoking the NiftiImage constructor with the noise dataset as argument:: @@ -352,7 +357,7 @@ will evaluate to True. -To save the noise file to disk, just call the save() method:: +To save the noise file to disk, just call the ``save()`` method:: nim.save('noise.nii.gz') @@ -383,7 +388,7 @@ Let's load another 4d NIfTI file and perform a linear detrending, by fitting a straight line to the timeseries of each voxel and substract that fit from the data. Although this might sound complicated at first, thanks to the -excellent SciPy module it is just a few lines of code:: +excellent SciPy_ module it is just a few lines of code:: nim = NiftiImage('timeseries.nii') @@ -453,7 +458,7 @@ to view a slice from a 3d volume. This time I assume that a 3d nifti file is already opened and available in the -nim3d object. At first we need to load the necessary Python module:: +``nim3d`` object. At first we need to load the necessary Python module:: from pylab import * @@ -462,7 +467,7 @@ imshow(nim3d.data[200], interpolation='nearest', cmap=cm.gray) show() -It is necessary to call the show() function one time after importing pylab +It is necessary to call the ``show()`` function one time after importing pylab to actually see the image when running Python interactively (`screenshot xyslice `_ ). @@ -504,9 +509,10 @@ This computes a peristimulus timeseries using the preprocessed fMRI from a FEAT output directory and two custom EV files that both together make up -condition A. --times indicates that the EV files list onset times (not volume -ids) and --nvols requests the mean peristimulus timecourse for 4 volumes after -stimulus onset (5 including onset). -p recodes the peristimulus timeseries into +condition A.``--times`` indicates that the EV files list onset times +(not volume ids) and ``--nvols`` requests the mean peristimulus timecourse +for 4 volumes after stimulus onset (5 including onset). +``-p`` recodes the peristimulus timeseries into percent signalchange, where the onset is always zero and any following value is the signal change with respect to the onset volume. From scipy-svn at scipy.org Mon Oct 8 18:54:56 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 8 Oct 2007 17:54:56 -0500 (CDT) Subject: [Scipy-svn] r3426 - trunk/scipy/io Message-ID: <20071008225456.CFCD339C0CC@new.scipy.org> Author: jarrod.millman Date: 2007-10-08 17:54:53 -0500 (Mon, 08 Oct 2007) New Revision: 3426 Removed: trunk/scipy/io/nifti/ Modified: trunk/scipy/io/__init__.py trunk/scipy/io/setup.py Log: removing nifti code Modified: trunk/scipy/io/__init__.py =================================================================== --- trunk/scipy/io/__init__.py 2007-10-08 19:43:45 UTC (rev 3425) +++ trunk/scipy/io/__init__.py 2007-10-08 22:54:53 UTC (rev 3426) @@ -17,10 +17,6 @@ from data_store import * from pickler import * -# Comment out until we resolve nifti build on Windows. -#from scipy.io.nifti import NiftiImage -#del nifti - from mmio import mminfo,mmread,mmwrite __all__ = filter(lambda s:not s.startswith('_'),dir()) Modified: trunk/scipy/io/setup.py =================================================================== --- trunk/scipy/io/setup.py 2007-10-08 19:43:45 UTC (rev 3425) +++ trunk/scipy/io/setup.py 2007-10-08 22:54:53 UTC (rev 3426) @@ -4,9 +4,6 @@ from numpy.distutils.misc_util import Configuration config = Configuration('io', parent_package, top_path) - # Comment out until we resolve nifti build on Windows. - #config.add_subpackage('nifti') - config.add_extension('numpyio', sources = ['numpyiomodule.c']) From scipy-svn at scipy.org Tue Oct 9 17:30:34 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 9 Oct 2007 16:30:34 -0500 (CDT) Subject: [Scipy-svn] r3427 - in trunk/scipy/sandbox/multigrid: . multigridtools tests Message-ID: <20071009213034.30EC439C06A@new.scipy.org> Author: wnbell Date: 2007-10-09 16:30:30 -0500 (Tue, 09 Oct 2007) New Revision: 3427 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/multigridtools/smoothed_aggregation.h trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/tests/test_sa.py trunk/scipy/sandbox/multigrid/utils.py Log: fixed bug in approximate_spectral_radius where A was assumed to be symmetric adaptive SA works for symmetric diagonal scaled case now Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-08 22:54:53 UTC (rev 3426) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-09 21:30:30 UTC (rev 3427) @@ -4,7 +4,7 @@ from relaxation import gauss_seidel from multilevel import multilevel_solver -from sa import sa_constant_interpolation +from sa import sa_constant_interpolation,sa_fit_candidates #from utils import infinity_norm from utils import approximate_spectral_radius @@ -144,7 +144,7 @@ Ps = [] for W in Ws: - P,x = fit_candidates(W,x) + P,x = sa_fit_candidates(W,x) I = smoothed_prolongator(P,A) A = I.T.tocsr() * A * I As.append(A) @@ -301,7 +301,8 @@ while len(AggOps) + 1 < max_levels and A_l.shape[0] > max_coarse: #W_l = sa_constant_interpolation(A_l,epsilon=0.08*0.5**(len(AggOps)-1)) #step 4b #TEST W_l = sa_constant_interpolation(A_l,epsilon=0) #step 4b - P_l,x = fit_candidate(W_l,x) #step 4c + P_l,x = sa_fit_candidates(W_l,[x]) #step 4c + x = x[0] #TODO make sa_fit_candidates accept a single x I_l = smoothed_prolongator(P_l,A_l) #step 4d A_l = I_l.T.tocsr() * A_l * I_l #step 4e @@ -337,15 +338,15 @@ from scipy import * from utils import diag_sparse from multilevel import poisson_problem1D,poisson_problem2D -A = poisson_problem2D(50) +A = poisson_problem2D(200) #A = io.mmread("tests/sample_data/laplacian_41_3dcube.mtx").tocsr() #A = io.mmread("laplacian_40_3dcube.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() #A = A*A -#D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() -#A = D * A * D +D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() +A = D * A * D #A = io.mmread("nos2.mtx").tocsr() asa = adaptive_sa_solver(A,max_candidates=1) #x = arange(A.shape[0]).astype('d') + 1 Modified: trunk/scipy/sandbox/multigrid/multigridtools/smoothed_aggregation.h =================================================================== --- trunk/scipy/sandbox/multigrid/multigridtools/smoothed_aggregation.h 2007-10-08 22:54:53 UTC (rev 3426) +++ trunk/scipy/sandbox/multigrid/multigridtools/smoothed_aggregation.h 2007-10-09 21:30:30 UTC (rev 3427) @@ -42,18 +42,26 @@ T eps_Aii = epsilon*epsilon*diags[i]; + T weak_sum = 0.0; + for(int jj = row_start; jj < row_end; jj++){ const int j = Aj[jj]; const T Aij = Ax[jj]; - if(i == j){continue;} + if(i == j){continue;} //skip diagonal until end of row // |A(i,j)| < epsilon * sqrt(|A(i,i)|*|A(j,j)|) if(Aij*Aij >= std::abs(eps_Aii * diags[j])){ Sj->push_back(j); Sx->push_back(Aij); + } else { + weak_sum += Aij; } } + //Add modified diagonal entry + Sj->push_back(i); + Sx->push_back(diags[i] + weak_sum); //filtered matrix + Sp->push_back(Sj->size()); } } Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-08 22:54:53 UTC (rev 3426) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-09 21:30:30 UTC (rev 3427) @@ -4,8 +4,9 @@ import scipy import numpy -from numpy import zeros,zeros_like,array +from numpy import ones,zeros,zeros_like,array from numpy.linalg import norm +from scipy.linsolve import spsolve from sa import sa_interpolation from rs import rs_interpolation @@ -19,8 +20,8 @@ with standard 3-point finite difference stencil on a grid with N points. """ - D = 2*numpy.ones(N) - O = -numpy.ones(N) + D = 2*ones(N) + O = -ones(N) return scipy.sparse.spdiags([D,O,O],[0,-1,1],N,N).tocoo().tocsr() #eliminate zeros def poisson_problem2D(N): @@ -29,9 +30,9 @@ with standard 5-point finite difference stencil on a square N-by-N grid. """ - D = 4*numpy.ones(N*N) - T = -numpy.ones(N*N) - O = -numpy.ones(N*N) + D = 4*ones(N*N) + T = -ones(N*N) + O = -ones(N*N) T[N-1::N] = 0 return scipy.sparse.spdiags([D,O,T,T,O],[0,-N,-1,1,N],N*N,N*N).tocoo().tocsr() #eliminate zeros @@ -130,12 +131,12 @@ #TODO change use of tol (relative tolerance) to agree with other iterative solvers A = self.As[0] - residuals = [scipy.linalg.norm(b-A*x)] + residuals = [ norm(b-A*x) ] while len(residuals) <= maxiter and residuals[-1]/residuals[0] > tol: self.__solve(0,x,b) - residuals.append(scipy.linalg.norm(b-A*x)) + residuals.append( norm(b-A*x) ) if callback is not None: callback(x) @@ -150,7 +151,7 @@ A = self.As[lvl] if len(self.As) == 1: - x[:] = scipy.linsolve.spsolve(A,b) + x[:] = spsolve(A,b) return self.presmoother(A,x,b) @@ -162,7 +163,7 @@ if lvl == len(self.As) - 2: #use direct solver on coarsest level - coarse_x[:] = scipy.linsolve.spsolve(self.As[-1],coarse_b) + coarse_x[:] = spsolve(self.As[-1],coarse_b) #coarse_x[:] = scipy.linalg.cg(self.As[-1],coarse_b,tol=1e-12)[0] #print "coarse residual norm",scipy.linalg.norm(coarse_b - self.As[-1]*coarse_x) else: Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-08 22:54:53 UTC (rev 3426) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-09 21:30:30 UTC (rev 3427) @@ -1,43 +1,60 @@ from numpy.testing import * -from numpy import sqrt,empty,ones,arange,array_split,eye,array,zeros,diag +from numpy import sqrt,empty,ones,arange,array_split,eye,array, \ + zeros,diag,zeros_like +from numpy.linalg import norm from scipy import rand -from scipy.sparse import spdiags,csr_matrix,lil_matrix +from scipy.sparse import spdiags,csr_matrix,lil_matrix, \ + isspmatrix_csr,isspmatrix_csc,isspmatrix_coo, \ + isspmatrix_lil import numpy set_package_path() import scipy.sandbox.multigrid from scipy.sandbox.multigrid.sa import sa_strong_connections, sa_constant_interpolation, \ sa_interpolation, sa_fit_candidates -from scipy.sandbox.multigrid.multilevel import poisson_problem1D,poisson_problem2D +from scipy.sandbox.multigrid.multilevel import poisson_problem1D,poisson_problem2D, \ + smoothed_aggregation_solver +from scipy.sandbox.multigrid.utils import diag_sparse restore_path() +#def sparsity(A): +# A = A.copy() +# +# if isspmatrix_csr(A) or isspmatrix_csc(A) or isspmatrix_coo(A): +# A.data[:] = 1 +# elif isspmatrix_lil: +# for row in A.data: +# row[:] = [1]*len(row) +# else: +# raise ValueError,'expected csr,csc,coo, or lil' +# +# return A + def reference_sa_strong_connections(A,epsilon): A_coo = A.tocoo() S = lil_matrix(A.shape) for (i,j,v) in zip(A_coo.row,A_coo.col,A_coo.data): - if i == j: continue #skip diagonal + if i == j or abs(v) >= epsilon*sqrt(abs(A[i,i])*abs(A[j,j])): + S[i,j] += v + else: + S[i,i] += v - if abs(A[i,j]) >= epsilon*sqrt(abs(A[i,i])*abs(A[j,j])): - S[i,j] = v + return S - return S.tocsr() - class TestSAStrongConnections(NumpyTestCase): - def check_simple(self): - N = 4 - A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() - S = spdiags([ -ones(N),-ones(N)],[-1,1],N,N).tocsr() - assert_array_equal(sa_strong_connections(A,0.50).todense(),S.todense()) #all connections are strong - assert_array_equal(sa_strong_connections(A,0.51).todense(),0*S.todense()) #no connections are strong - - N = 100 - A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() - S = spdiags([ -ones(N),-ones(N)],[-1,1],N,N).tocsr() - assert_array_equal(sa_strong_connections(A,0.50).todense(),S.todense()) #all connections are strong - assert_array_equal(sa_strong_connections(A,0.51).todense(),0*S.todense()) #no connections are strong +# def check_simple(self): +# N = 4 +# A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() +# assert_array_equal(sa_strong_connections(A,0.50).todense(),A.todense()) #all connections are strong +# assert_array_equal(sa_strong_connections(A,0.51).todense(),0*A.todense()) #no connections are strong +# +# N = 100 +# A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() +# assert_array_equal(sa_strong_connections(A,0.50).todense(),A.todense()) #all connections are strong +# assert_array_equal(sa_strong_connections(A,0.51).todense(),0*A.todense()) #no connections are strong def check_random(self): numpy.random.seed(0) @@ -47,7 +64,8 @@ for epsilon in [0.0,0.1,0.5,1.0,10.0]: S_result = sa_strong_connections(A,epsilon) S_expected = reference_sa_strong_connections(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) + assert_almost_equal(S_result.todense(),S_expected.todense()) + #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) def check_poisson1D(self): for N in [2,3,5,7,10,11,19]: @@ -56,6 +74,7 @@ S_result = sa_strong_connections(A,epsilon) S_expected = reference_sa_strong_connections(A,epsilon) assert_array_equal(S_result.todense(),S_expected.todense()) + #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) def check_poisson2D(self): for N in [2,3,5,7,10,11]: @@ -64,6 +83,7 @@ S_result = sa_strong_connections(A,epsilon) S_expected = reference_sa_strong_connections(A,epsilon) assert_array_equal(S_result.todense(),S_expected.todense()) + #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) @@ -209,7 +229,89 @@ +class TestSASolver(NumpyTestCase): + def setUp(self): + self.cases = [] + #self.cases.append((poisson_problem1D(10),None)) + + self.cases.append((poisson_problem1D(500),None)) + self.cases.append((poisson_problem2D(50),None)) + + def check_basic(self): + """check that method converges at a reasonable rate""" + + for A,candidates in self.cases: + ml = smoothed_aggregation_solver(A,candidates,max_coarse=10,max_levels=10) + + numpy.random.seed(0) #make tests repeatable + + x = rand(A.shape[0]) + b = A*rand(A.shape[0]) #zeros_like(x) + + x_sol,residuals = ml.solve(b,x0=x,maxiter=20,tol=1e-12,return_residuals=True) + + avg_convergence_ratio = (residuals[-1]/residuals[0])**(1.0/len(residuals)) + + assert(avg_convergence_ratio < 0.5) + + def check_DAD(self): + + for A,candidates in self.cases: + + x = rand(A.shape[0]) + b = A*rand(A.shape[0]) #zeros_like(x) + + D = diag_sparse(rand(A.shape[0])) + D_inv = diag_sparse(1.0/D.data) + DAD = D*A*D + + if candidates is None: + candidates = [ ones(A.shape[0]) ] + + DAD_candidates = [ (D_inv * c) for c in candidates ] + + #ml = smoothed_aggregation_solver(A,candidates,max_coarse=1,max_levels=2) + + ml = smoothed_aggregation_solver(DAD,DAD_candidates,max_coarse=100,max_levels=2) + + #print (D_inv*ml.Ps[0]).todense() + + x_sol,residuals = ml.solve(b,x0=x,maxiter=10,tol=1e-12,return_residuals=True) + + avg_convergence_ratio = (residuals[-1]/residuals[0])**(1.0/len(residuals)) + print avg_convergence_ratio + + assert(avg_convergence_ratio < 0.5) + +## def check_DAD(self): +## """check that method is invariant to symmetric diagonal scaling (A -> DAD)""" +## +## for A,A_candidates in self.cases: +## numpy.random.seed(0) #make tests repeatable +## +## x = rand(A.shape[0]) +## b = zeros_like(x) +## +## D = diag_sparse(rand(A.shape[0])) +## D_inv = diag_sparse(1.0/D.data) +## DAD = D*A*D +## +## +## if A_candidates is None: +## A_candidates = [ ones(A.shape[0]) ] +## +## DAD_candidates = [ (D_inv * c) for c in A_candidates ] +## +## ml_A = smoothed_aggregation_solver(A, A_candidates, max_coarse=10, max_levels=10, epsilon=0.0) +## x_sol_A = ml_A.solve(b, x0=x, maxiter=1, tol=1e-12) +## +## ml_DAD = smoothed_aggregation_solver(DAD, DAD_candidates, max_coarse=10, max_levels=10, epsilon=0.0) +## x_sol_DAD = ml_DAD.solve(b, x0=D*x, maxiter=1, tol=1e-12) +## +## assert_almost_equal(x_sol_A, D_inv * x_sol_DAD) + + if __name__ == '__main__': NumpyTest().run() Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-08 22:54:53 UTC (rev 3426) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-09 21:30:30 UTC (rev 3427) @@ -10,8 +10,8 @@ """ Approximate the spectral radius of a symmetric matrix using ARPACK """ - from scipy.sandbox.arpack import eigen_symmetric - return eigen_symmetric(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False)[0] + from scipy.sandbox.arpack import eigen + return eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False)[0] From scipy-svn at scipy.org Tue Oct 9 19:32:49 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 9 Oct 2007 18:32:49 -0500 (CDT) Subject: [Scipy-svn] r3428 - in trunk/scipy/ndimage: . src tests Message-ID: <20071009233249.67E9239C061@new.scipy.org> Author: stefan Date: 2007-10-09 18:32:25 -0500 (Tue, 09 Oct 2007) New Revision: 3428 Modified: trunk/scipy/ndimage/_ni_support.py trunk/scipy/ndimage/src/ni_interpolation.c trunk/scipy/ndimage/src/ni_support.h trunk/scipy/ndimage/tests/test_ndimage.py Log: Fix off-by-on errors in ndimage boundary routines. Update tests. Explicitly define enumeration. Modified: trunk/scipy/ndimage/_ni_support.py =================================================================== --- trunk/scipy/ndimage/_ni_support.py 2007-10-09 21:30:30 UTC (rev 3427) +++ trunk/scipy/ndimage/_ni_support.py 2007-10-09 23:32:25 UTC (rev 3428) @@ -41,7 +41,7 @@ elif mode == 'reflect': return 2 elif mode == 'mirror': - return 3 + return 3 elif mode == 'constant': return 4 else: Modified: trunk/scipy/ndimage/src/ni_interpolation.c =================================================================== --- trunk/scipy/ndimage/src/ni_interpolation.c 2007-10-09 21:30:30 UTC (rev 3427) +++ trunk/scipy/ndimage/src/ni_interpolation.c 2007-10-09 23:32:25 UTC (rev 3428) @@ -130,7 +130,7 @@ if (len <= 1) { in = 0; } else { - maybelong sz2 = 2 * len - 1; + maybelong sz2 = 2 * len; if (in < -sz2) in = sz2 * (maybelong)(-in / sz2) + in; in = in < -len ? in + sz2 : -in - 1; @@ -140,7 +140,7 @@ if (len <= 1) { in = 0; } else { - maybelong sz = len; + maybelong sz = len - 1; // Integer division of -in/sz gives (-in mod sz) // Note that 'in' is negative in += sz * ((maybelong)(-in / sz) + 1); @@ -179,7 +179,7 @@ if (len <= 1) { in = 0; } else { - maybelong sz = len; + maybelong sz = len - 1; in -= sz * (maybelong)(in / sz); } break; Modified: trunk/scipy/ndimage/src/ni_support.h =================================================================== --- trunk/scipy/ndimage/src/ni_support.h 2007-10-09 21:30:30 UTC (rev 3427) +++ trunk/scipy/ndimage/src/ni_support.h 2007-10-09 23:32:25 UTC (rev 3428) @@ -44,10 +44,10 @@ typedef enum { NI_EXTEND_FIRST = 0, NI_EXTEND_NEAREST = 0, - NI_EXTEND_WRAP, - NI_EXTEND_REFLECT, - NI_EXTEND_MIRROR, - NI_EXTEND_CONSTANT, + NI_EXTEND_WRAP = 1, + NI_EXTEND_REFLECT = 2, + NI_EXTEND_MIRROR = 3, + NI_EXTEND_CONSTANT = 4, NI_EXTEND_LAST = NI_EXTEND_CONSTANT, NI_EXTEND_DEFAULT = NI_EXTEND_MIRROR } NI_ExtendMode; Modified: trunk/scipy/ndimage/tests/test_ndimage.py =================================================================== --- trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-09 21:30:30 UTC (rev 3427) +++ trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-09 23:32:25 UTC (rev 3428) @@ -76,7 +76,7 @@ numpy.float32, numpy.float64] # list of boundary modes: - self.modes = ['nearest', 'wrap', 'reflect', 'constant'] + self.modes = ['nearest', 'wrap', 'reflect', 'mirror', 'constant'] def test_correlate01(self): "correlation 1" @@ -1166,11 +1166,12 @@ true_values = [[1, 1, 2], [3, 1, 2], [1, 1, 2], + [2, 1, 2], [0, 1, 2]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate1d(array, weights, 0, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output,true_value) def test_extend02(self): "line extension 2" @@ -1179,11 +1180,12 @@ true_values = [[1, 1, 1], [3, 1, 2], [3, 3, 2], + [1, 2, 3], [0, 0, 0]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate1d(array, weights, 0, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_extend03(self): "line extension 3" @@ -1192,11 +1194,12 @@ true_values = [[2, 3, 3], [2, 3, 1], [2, 3, 3], + [2, 3, 2], [2, 3, 0]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate1d(array, weights, 0, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_extend04(self): "line extension 4" @@ -1205,27 +1208,29 @@ true_values = [[3, 3, 3], [2, 3, 1], [2, 1, 1], + [1, 2, 3], [0, 0, 0]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate1d(array, weights, 0, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_extend05(self): "line extension 5" array = numpy.array([[1, 2, 3], - [4, 5, 6], - [7, 8, 9]]) + [4, 5, 6], + [7, 8, 9]]) weights = numpy.array([[1, 0], [0, 0]]) true_values = [[[1, 1, 2], [1, 1, 2], [4, 4, 5]], [[9, 7, 8], [3, 1, 2], [6, 4, 5]], [[1, 1, 2], [1, 1, 2], [4, 4, 5]], + [[5, 4, 5], [2, 1, 2], [5, 4, 5]], [[0, 0, 0], [0, 1, 2], [0, 4, 5]]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate(array, weights, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_extend06(self): @@ -1237,11 +1242,12 @@ true_values = [[[5, 6, 6], [8, 9, 9], [8, 9, 9]], [[5, 6, 4], [8, 9, 7], [2, 3, 1]], [[5, 6, 6], [8, 9, 9], [8, 9, 9]], + [[5, 6, 5], [8, 9, 8], [5, 6, 5]], [[5, 6, 0], [8, 9, 0], [0, 0, 0]]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate(array, weights, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_extend07(self): @@ -1251,11 +1257,12 @@ true_values = [[3, 3, 3], [2, 3, 1], [2, 1, 1], + [1, 2, 3], [0, 0, 0]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate(array, weights, mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + assert_array_equal(output, true_value) def test_extend08(self): "line extension 8" @@ -1265,11 +1272,12 @@ true_values = [[[3], [3], [3]], [[2], [3], [1]], [[2], [1], [1]], + [[1], [2], [3]], [[0], [0], [0]]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate(array, weights, mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + assert_array_equal(output, true_value) def test_extend09(self): "line extension 9" @@ -1278,11 +1286,12 @@ true_values = [[3, 3, 3], [2, 3, 1], [2, 1, 1], + [1, 2, 3], [0, 0, 0]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate(array, weights, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_extend10(self): "line extension 10" @@ -1292,29 +1301,47 @@ true_values = [[[3], [3], [3]], [[2], [3], [1]], [[2], [1], [1]], + [[1], [2], [3]], [[0], [0], [0]]] for mode, true_value in zip(self.modes, true_values): output = ndimage.correlate(array, weights, - mode = mode, cval = 0) - self.failUnless(diff(output, true_value) < eps) + mode = mode, cval = 0) + assert_array_equal(output, true_value) def test_boundaries(self): "boundary modes" def shift(x): - return (x[0] + 0.1,) + return (x[0] + 0.5,) + data = numpy.array([1,2,3,4.]) + expected = {'constant': [1.5,2.5,3.5,-1,-1,-1,-1], + 'wrap': [1.5,2.5,3.5,1.5,2.5,3.5,1.5], + 'mirror' : [1.5,2.5,3.5,3.5,2.5,1.5,1.5], + 'nearest' : [1.5,2.5,3.5,4,4,4,4]} + + for mode in expected.keys(): + assert_array_equal(expected[mode], + ndimage.geometric_transform(data,shift, + cval=-1,mode=mode, + output_shape=(7,), + order=1)) + + def test_boundaries2(self): + "boundary modes 2" + def shift(x): + return (x[0] - 0.9,) + data = numpy.array([1,2,3,4]) - expected = {'constant': [1,2,3,-1,-1,-1], - 'wrap': [1,2,3,4,1,2], - 'reflect' : [1,2,3,4,4,3], - 'mirror' : [1,2,3,4,3,2], - 'nearest' : [1,2,3,4,4,4]} + expected = {'constant': [-1,1,2,3], + 'wrap': [3,1,2,3], + 'mirror' : [2,1,2,3], + 'nearest' : [1,1,2,3]} for mode in expected.keys(): assert_array_equal(expected[mode], ndimage.geometric_transform(data,shift, cval=-1,mode=mode, - output_shape=(6,))) + output_shape=(4,))) def test_fourier_gaussian_real01(self): "gaussian fourier filter for real transforms 1" @@ -2247,7 +2274,7 @@ out = ndimage.rotate(data, 90) for i in range(3): self.failUnless(diff(out[:,:,i], true) < eps) - + def test_rotate06(self): "rotate 6" data = numpy.empty((3,4,3)) @@ -2265,7 +2292,7 @@ out = ndimage.rotate(data, 90) for i in range(3): self.failUnless(diff(out[:,:,i], true) < eps) - + def test_rotate07(self): "rotate 7" data = numpy.array([[[0, 0, 0, 0, 0], From scipy-svn at scipy.org Tue Oct 9 20:02:56 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 9 Oct 2007 19:02:56 -0500 (CDT) Subject: [Scipy-svn] r3429 - trunk/scipy/io Message-ID: <20071010000256.6117639C060@new.scipy.org> Author: chris.burns Date: 2007-10-09 19:02:53 -0500 (Tue, 09 Oct 2007) New Revision: 3429 Modified: trunk/scipy/io/datasource.py Log: Cleanup the public interface and document. Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-09 23:32:25 UTC (rev 3428) +++ trunk/scipy/io/datasource.py 2007-10-10 00:02:53 UTC (rev 3429) @@ -1,8 +1,12 @@ -"""Utilities for importing (possibly compressed) data sets from an URL -(or file) and possibly caching them. +"""A generic interface for importing data. Data sources can originate from +URLs or local paths and can be in compressed or uncompressed form. """ +# TODO: Make DataSource and Repository the public interface. +# Cache will be used internally. Add methods in DataSource to expose +# some of the functionality that exists only in Cache currently. + __docformat__ = "restructuredtext en" import os @@ -31,8 +35,8 @@ warnings.warn(_api_warning) # TODO: .zip support -zipexts = (".gz",".bz2") -file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file} +_zipexts = (".gz",".bz2") +_file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file} def iszip(filename): """Test if the given file is a zip file. @@ -50,7 +54,7 @@ """ _tmp, ext = path(filename).splitext() - return ext in zipexts + return ext in _zipexts def unzip(filename): """Unzip the given file and return the path object to the new file. @@ -70,7 +74,7 @@ if not iszip(filename): raise ValueError("file %s is not zipped"%filename) unzip_name, zipext = splitzipext(filename) - opener = file_openers[zipext] + opener = _file_openers[zipext] outfile = file(unzip_name, 'w') outfile.write(opener(filename).read()) outfile.close() @@ -119,7 +123,7 @@ else: return filename, None -def isurl(pathstr): +def _isurl(pathstr): """Test whether a given string can be parsed as a URL. *Parameters*: @@ -133,7 +137,7 @@ """ - scheme, netloc, _tmp, _tmp, _tmp, _tmp = urlparse(pathstr) + scheme, netloc, _path, _params, _query, _frag = urlparse(pathstr) return bool(scheme and netloc) def ensuredirs(directory): @@ -233,7 +237,7 @@ >>> mycache = datasource.Cache() >>> mycache.filename('xyzcoords.txt') - '/home/guido/.scipy/cache/yzcoords.txt' + '/home/guido/.scipy/cache/xyzcoords.txt' """ # TODO: Change to non-public? @@ -263,13 +267,15 @@ file(upath, 'w').write(openedurl.read()) def clear(self): - """ Delete all files in the cache. + """Delete all files in the cache.""" - :Returns: ``None`` - """ + # TODO: This deletes all files in the cache directory, regardless + # of if this instance created them. Too destructive and + # unexpected behavior. + for _file in self.path.files(): - _file.rm() - + os.remove(file) + def iscached(self, uri): """ Check if a file exists in the cache. @@ -303,25 +309,54 @@ self._cache = Cache(cachepath) def tempfile(self, suffix='', prefix=''): - """Return an temporary file name in the cache.""" + """Create a temporary file in the cache. + + *Parameters*: + suffix : {''}, optional + + prefix : {''}, optional + + *Returns*: + tmpfile : {string} + String containing the full path to the temporary file. + + *Examples* + + >>> datasrc = datasource.DataSource() + >>> tmpfile = datasrc.tempfile() + >>> tmpfile + '/home/guido/src/scipy-trunk/scipy/io/PZTuKo' + + """ return self._cache.tempfile(suffix, prefix) def _possible_names(self, filename): + """Return a tuple containing compressed filenames.""" names = [filename] if not iszip(filename): - for zipext in zipexts: + for zipext in _zipexts: names.append(filename+zipext) return tuple(names) def cache(self, pathstr): - if isurl(pathstr): + # TODO: Should work with files also, not just urls. + if _isurl(pathstr): self._cache.cache(pathstr) + def clear(self): + # TODO: Implement a clear interface for deleting tempfiles. + # There's a problem with the way this is handled in the Cache, + # All files in the cache directory will be deleted. In the + # default instance, that's the os.curdir. I doubt this is what + # people would want. The instance should only delete files that + # it created! + pass + def filename(self, pathstr): found = None for name in self._possible_names(pathstr): try: - if isurl(name): + if _isurl(name): self.cache(name) found = self._cache.filename(name) else: @@ -343,13 +378,13 @@ return False def open(self, pathstr, mode='r'): - if isurl(pathstr) and iswritemode(mode): + if _isurl(pathstr) and iswritemode(mode): raise ValueError("URLs are not writeable") found = self.filename(pathstr) _, ext = splitzipext(found) if ext == 'bz2': mode.replace("+", "") - return file_openers[ext](found, mode=mode) + return _file_openers[ext](found, mode=mode) def _fullpath(self, pathstr): return pathstr From scipy-svn at scipy.org Tue Oct 9 20:33:52 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 9 Oct 2007 19:33:52 -0500 (CDT) Subject: [Scipy-svn] r3430 - trunk/scipy/sparse Message-ID: <20071010003352.B029B39C1CD@new.scipy.org> Author: stefan Date: 2007-10-09 19:33:40 -0500 (Tue, 09 Oct 2007) New Revision: 3430 Modified: trunk/scipy/sparse/sparse.py Log: Remove type checking from sparse array constructor. Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-10 00:02:53 UTC (rev 3429) +++ trunk/scipy/sparse/sparse.py 2007-10-10 00:33:40 UTC (rev 3430) @@ -272,7 +272,7 @@ def __imul__(self, other): raise NotImplementedError - + def __idiv__(self, other): return self.__itruediv__(other) @@ -635,7 +635,7 @@ return self._binopt(other,fn) else: raise NotImplementedError - + def __itruediv__(self, other): #self *= other if isscalarlike(other): recip = 1.0 / other @@ -779,7 +779,7 @@ i1 = num + i1 return i0, i1 - + elif isscalar( sl ): if sl < 0: sl += num @@ -1933,7 +1933,7 @@ except AttributeError: tr = asarray(other).transpose() return self.transpose().dot(tr).transpose() - + def __truediv__(self, other): # self * other if isscalarlike(other): new = dok_matrix(self.shape, dtype=self.dtype) @@ -1945,7 +1945,7 @@ else: return self.tocsr() / other - + def __itruediv__(self, other): # self * other if isscalarlike(other): # Multiply this scalar by every element. @@ -2788,12 +2788,6 @@ else: newdtype = numpy.dtype(dtype) - allowed = 'fdFD' - if newdtype.char not in allowed: - if default is None or canCast: - newdtype = numpy.dtype( 'd' ) - else: - raise TypeError, "dtype must be one of 'fdFD'" return newdtype From scipy-svn at scipy.org Wed Oct 10 13:12:39 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 10 Oct 2007 12:12:39 -0500 (CDT) Subject: [Scipy-svn] r3431 - trunk/scipy/sandbox/multigrid Message-ID: <20071010171239.CFF27C7C054@new.scipy.org> Author: wnbell Date: 2007-10-10 12:12:37 -0500 (Wed, 10 Oct 2007) New Revision: 3431 Added: trunk/scipy/sandbox/multigrid/rs.py trunk/scipy/sandbox/multigrid/sa.py Modified: trunk/scipy/sandbox/multigrid/adaptive.py Log: forgot to include rs.py and sa.py Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-10 00:33:40 UTC (rev 3430) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-10 17:12:37 UTC (rev 3431) @@ -345,8 +345,8 @@ #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() #A = A*A -D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() -A = D * A * D +#D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() +#A = D * A * D #A = io.mmread("nos2.mtx").tocsr() asa = adaptive_sa_solver(A,max_candidates=1) #x = arange(A.shape[0]).astype('d') + 1 Added: trunk/scipy/sandbox/multigrid/rs.py =================================================================== --- trunk/scipy/sandbox/multigrid/rs.py 2007-10-10 00:33:40 UTC (rev 3430) +++ trunk/scipy/sandbox/multigrid/rs.py 2007-10-10 17:12:37 UTC (rev 3431) @@ -0,0 +1,37 @@ +from scipy.sparse import csr_matrix,isspmatrix_csr + +import multigridtools + +__all__ = ['rs_strong_connections','rs_interpolation'] + +def rs_strong_connections(A,theta): + """Return a strength of connection matrix using the method of Ruge and Stuben + + An off-diagonal entry A[i.j] is a strong connection iff + + -A[i,j] >= theta * max( -A[i,k] ) where k != i + """ + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + + Sp,Sj,Sx = multigridtools.rs_strong_connections(A.shape[0],theta,A.indptr,A.indices,A.data) + return csr_matrix((Sx,Sj,Sp),dims=A.shape) + + +def rs_interpolation(A,theta=0.25): + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + + S = rs_strong_connections(A,theta) + + T = S.T.tocsr() #transpose S for efficient column access + + Ip,Ij,Ix = multigridtools.rs_interpolation(A.shape[0],\ + A.indptr,A.indices,A.data,\ + S.indptr,S.indices,S.data,\ + T.indptr,T.indices,T.data) + + return csr_matrix((Ix,Ij,Ip)) + + + + + Added: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-10 00:33:40 UTC (rev 3430) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-10 17:12:37 UTC (rev 3431) @@ -0,0 +1,125 @@ +import scipy +import numpy +from numpy import array,arange,ones,zeros,sqrt,isinf,asarray,empty +from scipy.sparse import csr_matrix,isspmatrix_csr + +from utils import diag_sparse,approximate_spectral_radius +import multigridtools + +__all__ = ['sa_strong_connections','sa_constant_interpolation', + 'sa_interpolation','sa_fit_candidates'] + + +def sa_strong_connections(A,epsilon): + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + + Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) + + #D = diag_sparse(D) + #I,J,V = arange(A.shape[0]).repeat(diff(A.indptr)),A.indices,A.data #COO format for A + #diag_mask = (I == J) + + return csr_matrix((Sx,Sj,Sp),A.shape) + +def sa_constant_interpolation(A,epsilon,blocks=None): + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + + #TODO handle epsilon = 0 case without creating strength of connection matrix? + + if blocks is not None: + num_dofs = A.shape[0] + num_blocks = blocks.max() + + if num_dofs != len(blocks): + raise ValueError,'improper block specification' + + # for non-scalar problems, use pre-defined blocks in aggregation + # the strength of connection matrix is based on the Frobenius norms of the blocks + + B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) + Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B #Frobenius norms of blocks entries of A + + S = sa_strong_connections(Block_Frob,epsilon) + + Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) + Pj = Pj[blocks] #expand block aggregates into constituent dofs + Pp = B.indptr + Px = B.data + else: + S = sa_strong_connections(A,epsilon) + + Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) + Pp = numpy.arange(len(Pj)+1) + Px = numpy.ones(len(Pj)) + + return csr_matrix((Px,Pj,Pp)) + + +def sa_fit_candidates(AggOp,candidates): + K = len(candidates) + + N_fine,N_coarse = AggOp.shape + + if K > 1 and len(candidates[0]) == K*N_fine: + #see if fine space has been expanded (all levels except for first) + AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) + N_fine = K*N_fine + + #TODO convert this to list of coarse candidates + R = zeros((K*N_coarse,K)) #storage for coarse candidates + + candidate_matrices = [] + for i,c in enumerate(candidates): + #TODO permit incomplete AggOps here (for k-form problems) (other modifications necessary?) + X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) + + + #orthogonalize X against previous + for j,A in enumerate(candidate_matrices): + D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X + R[j::K,i] = D_AtX + X.data -= D_AtX[X.indices] * A.data + + + #normalize X + D_XtX = csr_matrix((X.data**2,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of X.T * X + col_norms = sqrt(D_XtX) + R[i::K,i] = col_norms + col_norms = 1.0/col_norms + col_norms[isinf(col_norms)] = 0 + X.data *= col_norms[X.indices] + + candidate_matrices.append(X) + + Q_indptr = K*AggOp.indptr + Q_indices = (K*AggOp.indices).repeat(K) + for i in range(K): + Q_indices[i::K] += i + Q_data = empty(N_fine * K) + for i,X in enumerate(candidate_matrices): + Q_data[i::K] = X.data + Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) + + coarse_candidates = [array(R[:,i]) for i in range(K)] + + return Q,coarse_candidates + + +def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + + AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) + T,coarse_candidates = sa_fit_candidates(AggOp,candidates) + + #TODO use filtered matrix here for anisotropic problems + A_filtered = A + D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) + D_inv_A = D_inv * A_filtered + D_inv_A *= omega/approximate_spectral_radius(D_inv_A) + + P = T - (D_inv_A*T) + + return P,coarse_candidates + + + From scipy-svn at scipy.org Wed Oct 10 14:06:02 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 10 Oct 2007 13:06:02 -0500 (CDT) Subject: [Scipy-svn] r3432 - trunk/scipy/sandbox/multigrid Message-ID: <20071010180602.F297439C069@new.scipy.org> Author: wnbell Date: 2007-10-10 13:06:00 -0500 (Wed, 10 Oct 2007) New Revision: 3432 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/utils.py Log: fixed bug in approximate_spectral_radius Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-10 17:12:37 UTC (rev 3431) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-10 18:06:00 UTC (rev 3432) @@ -5,53 +5,9 @@ from relaxation import gauss_seidel from multilevel import multilevel_solver from sa import sa_constant_interpolation,sa_fit_candidates -#from utils import infinity_norm from utils import approximate_spectral_radius -##def fit_candidate(I,x): -## """ -## For each aggregate in I (i.e. each column of I) compute vector R and -## sparse matrix Q (having the sparsity of I) such that the following holds: -## -## Q*R = x and Q^T*Q = I -## -## In otherwords, find a prolongator Q with orthonormal columns so that -## x is represented exactly on the coarser level by R. -## """ -## x = asarray(x) -## Q = csr_matrix((x.copy(),I.indices,I.indptr),dims=I.shape,check=False) -## R = sqrt(ravel(csr_matrix((x*x,I.indices,I.indptr),dims=I.shape,check=False).sum(axis=0))) #column 2-norms -## -## Q.data *= (1.0/R)[Q.indices] #normalize columns of Q -## -## #print "norm(R)",scipy.linalg.norm(R) -## #print "min(R),max(R)",min(R),max(R) -## #print "infinity_norm(Q.T*Q - I) ",infinity_norm((Q.T.tocsr() * Q - scipy.sparse.spidentity(Q.shape[1]))) -## #print "norm(Q*R - x)",scipy.linalg.norm(Q*R - x) -## #print "norm(x - Q*Q.Tx)",scipy.linalg.norm(x - Q*(Q.T*x)) -## return Q,R - - - -##def orthonormalize_candidate(I,x,basis): -## Px = csr_matrix((x,I.indices,I.indptr),dims=I.shape,check=False) -## Rs = [] -## #othogonalize columns of Px against other candidates -## for b in basis: -## Pb = csr_matrix((b,I.indices,I.indptr),dims=I.shape,check=False) -## R = ravel(csr_matrix((Pb.data*Px.data,I.indices,I.indptr),dims=I.shape,check=False).sum(axis=0)) # columnwise projection of Px on Pb -## Px.data -= R[I.indices] * Pb.data #subtract component in b direction -## Rs.append(R) -## -## #filter columns here, set unused cols to 0, add to mask -## -## #normalize columns of Px -## R = ravel(csr_matrix((x**x,I.indices,I.indptr),dims=I.shape,check=False).sum(axis=0)) -## Px.data *= (1.0/R)[I.indices] -## Rs.append(R.reshape(-1,1)) -## return Rs - def hstack_csr(A,B): #OPTIMIZE THIS assert(A.shape[0] == B.shape[0]) @@ -327,7 +283,6 @@ for A_l,I in reversed(zip(As[1:],Is)): gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST x = I * x - gauss_seidel(A,x,b,iterations=mu) #TEST return x,AggOps #first candidate,aggregation @@ -344,28 +299,30 @@ #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() -#A = A*A + #D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() #A = D * A * D -#A = io.mmread("nos2.mtx").tocsr() -asa = adaptive_sa_solver(A,max_candidates=1) -#x = arange(A.shape[0]).astype('d') + 1 -scipy.random.seed(0) #TEST + +#A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() + +asa = adaptive_sa_solver(A,max_candidates=1,mu=5) +scipy.random.seed(0) #make tests repeatable x = rand(A.shape[0]) -b = zeros_like(x) +b = A*rand(A.shape[0]) print "solving" -#x_sol,residuals = asa.solver.solve(b,x,tol=1e-8,maxiter=30,return_residuals=True) -if True: +if False: x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=10,tol=1e-12,return_residuals=True) else: residuals = [] def add_resid(x): residuals.append(linalg.norm(b - A*x)) A.psolve = asa.solver.psolve - x_sol = linalg.cg(A,b,x0=x,maxiter=20,tol=1e-100,callback=add_resid)[0] + x_sol = linalg.cg(A,b,x0=x,maxiter=20,tol=1e-12,callback=add_resid)[0] + residuals = array(residuals)/residuals[0] + print "residuals ",residuals print "mean convergence factor",(residuals[-1]/residuals[0])**(1.0/len(residuals)) print "last convergence factor",residuals[-1]/residuals[-2] Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-10 17:12:37 UTC (rev 3431) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-10 18:06:00 UTC (rev 3432) @@ -1,7 +1,9 @@ __all__ =['approximate_spectral_radius','infinity_norm','diag_sparse'] -import numpy,scipy,scipy.sparse +import numpy +import scipy from numpy import ravel,arange +from scipy.linalg import norm from scipy.sparse import isspmatrix,isspmatrix_csr,isspmatrix_csc, \ csr_matrix,csc_matrix,extract_diagonal @@ -11,7 +13,7 @@ Approximate the spectral radius of a symmetric matrix using ARPACK """ from scipy.sandbox.arpack import eigen - return eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False)[0] + return norm(eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False)[0]) From scipy-svn at scipy.org Thu Oct 11 16:54:05 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 11 Oct 2007 15:54:05 -0500 (CDT) Subject: [Scipy-svn] r3433 - in trunk/scipy/sandbox/multigrid: . tests Message-ID: <20071011205405.13D9D39C038@new.scipy.org> Author: wnbell Date: 2007-10-11 15:53:54 -0500 (Thu, 11 Oct 2007) New Revision: 3433 Removed: trunk/scipy/sandbox/multigrid/coarsen.py Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/simple_test.py trunk/scipy/sandbox/multigrid/tests/test_sa.py trunk/scipy/sandbox/multigrid/utils.py Log: added more tests for aggregation operators that exclude nodes consolidated some SA tests Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-10 18:06:00 UTC (rev 3432) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-11 20:53:54 UTC (rev 3433) @@ -5,28 +5,9 @@ from relaxation import gauss_seidel from multilevel import multilevel_solver from sa import sa_constant_interpolation,sa_fit_candidates -from utils import approximate_spectral_radius +from utils import approximate_spectral_radius,hstack_csr,vstack_csr -def hstack_csr(A,B): - #OPTIMIZE THIS - assert(A.shape[0] == B.shape[0]) - A = A.tocoo() - B = B.tocoo() - I = concatenate((A.row,B.row)) - J = concatenate((A.col,B.col+A.shape[1])) - V = concatenate((A.data,B.data)) - return coo_matrix((V,(I,J)),dims=(A.shape[0],A.shape[1]+B.shape[1])).tocsr() - -def vstack_csr(A,B): - #OPTIMIZE THIS - assert(A.shape[1] == B.shape[1]) - A = A.tocoo() - B = B.tocoo() - I = concatenate((A.row,B.row+A.shape[0])) - J = concatenate((A.col,B.col)) - V = concatenate((A.data,B.data)) - return coo_matrix((V,(I,J)),dims=(A.shape[0]+B.shape[0],A.shape[1])).tocsr() @@ -34,7 +15,9 @@ """ """ - X = csr_matrix((x_l,W_l.indices,W_l.indptr),dims=W_l.shape,check=False) #candidate prolongator (assumes every value from x is used) + + #candidate prolongator (assumes every value from x is used) + X = csr_matrix((x_l,W_l.indices,W_l.indptr),dims=W_l.shape,check=False) R = (P_l.T.tocsr() * X) # R has at most 1 nz per row X = X - P_l*R # othogonalize X against P_l @@ -78,6 +61,7 @@ omega = 4.0/(3.0*approximate_spectral_radius(D_inv_A)) print "spectral radius",approximate_spectral_radius(D_inv_A) D_inv_A *= omega + return P - D_inv_A*P @@ -114,16 +98,21 @@ return csr_matrix((I.data,I.indices,ptr),dims=(N,I.shape[1]),check=False) class adaptive_sa_solver: - def __init__(self,A,options=None,max_levels=10,max_coarse=100,max_candidates=1,mu=5,epsilon=0.1): + def __init__(self,A,blocks=None,options=None,max_levels=10,max_coarse=100,\ + max_candidates=1,mu=5,epsilon=0.1): + self.A = A self.Rs = [] - #if self.A.shape[0] <= self.opts['coarse: max size']: - # raise ValueError,'small matrices not handled yet' + if self.A.shape[0] <= self.opts['coarse: max size']: + raise ValueError,'small matrices not handled yet' + + #first candidate + x,AggOps = self.__initialization_stage(A,blocks=blocks,\ + max_levels=max_levels,max_coarse=max_coarse,\ + mu=mu,epsilon=epsilon) - x,AggOps = self.__initialization_stage(A,max_levels=max_levels,max_coarse=max_coarse,mu=mu,epsilon=epsilon) #first candidate - Ws = AggOps self.candidates = [x] @@ -135,27 +124,72 @@ x = self.__develop_candidate(A,As,Is,Ps,Ws,AggOps,mu=mu) self.candidates.append(x) - - #if i == 0: - # x = arange(50).repeat(50).astype(float) - #elif i == 1: - # x = arange(50).repeat(50).astype(float) - # x = numpy.ravel(transpose(x.reshape((50,50)))) #As,Is,Ps,Ws = self.__augment_cycle(A,As,Ps,Ws,AggOps,x) As,Is,Ps = sa_hierarchy(A,AggOps,self.candidates) - - #random.seed(0) - #solver = multilevel_solver(As,Is) - #x = solver.solve(zeros(A.shape[0]), x0=rand(A.shape[0]), tol=1e-12, maxiter=30) - #self.candidates.append(x) self.Ps = Ps self.solver = multilevel_solver(As,Is) self.AggOps = AggOps + def __initialization_stage(self,A,max_levels,max_coarse,mu,epsilon): + AggOps = [] + Is = [] + # aSA parameters + # mu - number of test relaxation iterations + # epsilon - minimum acceptable relaxation convergence factor + + #step 1 + A_l = A + x = scipy.rand(A_l.shape[0]) + skip_f_to_i = False + + #step 2 + b = zeros_like(x) + gauss_seidel(A_l,x,b,iterations=mu,sweep='symmetric') + + #step 3 + #TODO test convergence rate here + + As = [A] + + while len(AggOps) + 1 < max_levels and A_l.shape[0] > max_coarse: + W_l = sa_constant_interpolation(A_l,epsilon=0,blocks=blocks) #step 4b + P_l,x = sa_fit_candidates(W_l,[x]) #step 4c + x = x[0] #TODO make sa_fit_candidates accept a single x + I_l = smoothed_prolongator(P_l,A_l) #step 4d + A_l = I_l.T.tocsr() * A_l * I_l #step 4e + + AggOps.append(W_l) + Is.append(I_l) + As.append(A_l) + + if A_l.shape <= max_coarse: break + + if not skip_f_to_i: + print "." + x_hat = x.copy() #step 4g + gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #step 4h + x_A_x = inner(x,A_l*x) + if (x_A_x/inner(x_hat,A_l*x_hat))**(1.0/mu) < epsilon: #step 4i + print "sufficient convergence, skipping" + skip_f_to_i = True + if x_A_x == 0: + x = x_hat #need to restore x + + #update fine-level candidate + for A_l,I in reversed(zip(As[1:],Is)): + gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST + x = I * x + gauss_seidel(A,x,b,iterations=mu) #TEST + + return x,AggOps #first candidate,aggregation + + + + def __develop_candidate(self,A,As,Is,Ps,Ws,AggOps,mu): #scipy.random.seed(0) #TEST x = scipy.rand(A.shape[0]) @@ -231,65 +265,7 @@ return new_As,new_Is,new_Ps,new_Ws - def __initialization_stage(self,A,max_levels,max_coarse,mu,epsilon): - AggOps = [] - Is = [] - # aSA parameters - # mu - number of test relaxation iterations - # epsilon - minimum acceptable relaxation convergence factor - - #scipy.random.seed(0) #TEST - - #step 1 - A_l = A - x = scipy.rand(A_l.shape[0]) - skip_f_to_i = False - - #step 2 - b = zeros_like(x) - gauss_seidel(A_l,x,b,iterations=mu,sweep='symmetric') - #step 3 - #test convergence rate here - - As = [A] - - while len(AggOps) + 1 < max_levels and A_l.shape[0] > max_coarse: - #W_l = sa_constant_interpolation(A_l,epsilon=0.08*0.5**(len(AggOps)-1)) #step 4b #TEST - W_l = sa_constant_interpolation(A_l,epsilon=0) #step 4b - P_l,x = sa_fit_candidates(W_l,[x]) #step 4c - x = x[0] #TODO make sa_fit_candidates accept a single x - I_l = smoothed_prolongator(P_l,A_l) #step 4d - A_l = I_l.T.tocsr() * A_l * I_l #step 4e - - AggOps.append(W_l) - Is.append(I_l) - As.append(A_l) - - if A_l.shape <= max_coarse: break - - if not skip_f_to_i: - print "." - x_hat = x.copy() #step 4g - gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #step 4h - x_A_x = inner(x,A_l*x) - if (x_A_x/inner(x_hat,A_l*x_hat))**(1.0/mu) < epsilon: #step 4i - print "sufficient convergence, skipping" - skip_f_to_i = True - if x_A_x == 0: - x = x_hat #need to restore x - - #update fine-level candidate - for A_l,I in reversed(zip(As[1:],Is)): - gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST - x = I * x - gauss_seidel(A,x,b,iterations=mu) #TEST - - return x,AggOps #first candidate,aggregation - - - - from scipy import * from utils import diag_sparse from multilevel import poisson_problem1D,poisson_problem2D Deleted: trunk/scipy/sandbox/multigrid/coarsen.py =================================================================== --- trunk/scipy/sandbox/multigrid/coarsen.py 2007-10-10 18:06:00 UTC (rev 3432) +++ trunk/scipy/sandbox/multigrid/coarsen.py 2007-10-11 20:53:54 UTC (rev 3433) @@ -1,163 +0,0 @@ -##import scipy -##import numpy -## -##from numpy import arange,ones,zeros,sqrt,isinf,asarray,empty -##from scipy.sparse import csr_matrix,isspmatrix_csr -## -##from utils import diag_sparse,approximate_spectral_radius -##import multigridtools -## -## -##def rs_strong_connections(A,theta): -## """ -## Return a strength of connection matrix using the method of Ruge and Stuben -## -## An off-diagonal entry A[i.j] is a strong connection iff -## -## -A[i,j] >= theta * max( -A[i,k] ) where k != i -## """ -## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') -## -## Sp,Sj,Sx = multigridtools.rs_strong_connections(A.shape[0],theta,A.indptr,A.indices,A.data) -## return csr_matrix((Sx,Sj,Sp),A.shape) -## -## -##def rs_interpolation(A,theta=0.25): -## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') -## -## S = rs_strong_connections(A,theta) -## -## T = S.T.tocsr() #transpose S for efficient column access -## -## Ip,Ij,Ix = multigridtools.rs_interpolation(A.shape[0],\ -## A.indptr,A.indices,A.data,\ -## S.indptr,S.indices,S.data,\ -## T.indptr,T.indices,T.data) -## -## return csr_matrix((Ix,Ij,Ip)) -## -## -##def sa_strong_connections(A,epsilon): -## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') -## -## Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) -## -## return csr_matrix((Sx,Sj,Sp),A.shape) -## -##def sa_constant_interpolation(A,epsilon,blocks=None): -## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') -## -## #handle epsilon = 0 case without creating strength of connection matrix? -## -## if blocks is not None: -## num_dofs = A.shape[0] -## num_blocks = blocks.max() -## -## if num_dofs != len(blocks): -## raise ValueError,'improper block specification' -## -## # for non-scalar problems, use pre-defined blocks in aggregation -## # the strength of connection matrix is based on the Frobenius norms of the blocks -## -## B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) -## Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B #Frobenius norms of blocks entries of A -## -## S = sa_strong_connections(Block_Frob,epsilon) -## -## Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) -## Pj = Pj[blocks] #expand block aggregates into constituent dofs -## Pp = B.indptr -## Px = B.data -## else: -## S = sa_strong_connections(A,epsilon) -## -## Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) -## Pp = numpy.arange(len(Pj)+1) -## Px = numpy.ones(len(Pj)) -## -## return csr_matrix((Px,Pj,Pp)) -## -## -##def fit_candidates(AggOp,candidates): -## K = len(candidates) -## -## N_fine,N_coarse = AggOp.shape -## -## if K > 1 and len(candidates[0]) == K*N_fine: -## #see if fine space has been expanded (all levels except for first) -## AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) -## N_fine = K*N_fine -## -## R = zeros((K*N_coarse,K)) -## -## candidate_matrices = [] -## for i,c in enumerate(candidates): -## X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) -## -## #TODO optimize this -## -## #orthogonalize X against previous -## for j,A in enumerate(candidate_matrices): -## D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X -## R[j::K,i] = D_AtX -## X.data -= D_AtX[X.indices] * A.data -## -## #AtX = csr_matrix(A.T.tocsr() * X -## #R[j::K,i] = AtX.data -## #X = X - A * AtX -## -## #normalize X -## XtX = X.T.tocsr() * X -## col_norms = sqrt(asarray(XtX.sum(axis=0)).flatten()) -## R[i::K,i] = col_norms -## col_norms = 1.0/col_norms -## col_norms[isinf(col_norms)] = 0 -## X.data *= col_norms[X.indices] -## -## candidate_matrices.append(X) -## -## -## Q_indptr = K*AggOp.indptr -## Q_indices = (K*AggOp.indices).repeat(K) -## for i in range(K): -## Q_indices[i::K] += i -## Q_data = empty(N_fine * K) -## for i,X in enumerate(candidate_matrices): -## Q_data[i::K] = X.data -## Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) -## -## coarse_candidates = [R[:,i] for i in range(K)] -## -## return Q,coarse_candidates -## -#### S = sa_strong_connections(A,epsilon) -#### -#### #tentative (non-smooth) interpolation operator I -#### Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) -#### Pp = numpy.arange(len(Pj)+1) -#### Px = numpy.ones(len(Pj)) -#### -#### return scipy.sparse.csr_matrix((Px,Pj,Pp)) -## -####def sa_smoother(A,S,omega): -#### Bp,Bj,Bx = multigridtools.sa_smoother(A.shape[0],omega,A.indptr,A.indices,A.data,S.indptr,S.indices,S.data) -#### -#### return csr_matrix((Bx,Bj,Bp),dims=A.shape) -## -##def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): -## if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') -## -## AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) -## T,coarse_candidates = fit_candidates(AggOp,candidates) -## -## D_inv = diag_sparse(1.0/diag_sparse(A)) -## -## D_inv_A = D_inv * A -## D_inv_A *= omega/approximate_spectral_radius(D_inv_A) -## -## P = T - (D_inv_A*T) #same as I=S*P, (faster?) -## -## return P,coarse_candidates -## -## -## Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-10 18:06:00 UTC (rev 3432) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-11 20:53:54 UTC (rev 3433) @@ -1,31 +1,64 @@ import scipy import numpy -from numpy import array,arange,ones,zeros,sqrt,isinf,asarray,empty +from numpy import array,arange,ones,zeros,sqrt,isinf,asarray,empty,diff from scipy.sparse import csr_matrix,isspmatrix_csr from utils import diag_sparse,approximate_spectral_radius import multigridtools -__all__ = ['sa_strong_connections','sa_constant_interpolation', +__all__ = ['sa_filtered_matrix','sa_strong_connections','sa_constant_interpolation', 'sa_interpolation','sa_fit_candidates'] -def sa_strong_connections(A,epsilon): +def sa_filtered_matrix(A,epsilon,blocks=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + if epsilon == 0: + A_filtered = A + + else: + if blocks is None: + Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) + A_filtered = csr_matrix((Sx,Sj,Sp),A.shape) + + else: + num_dofs = A.shape[0] + num_blocks = blocks.max() + + if num_dofs != len(blocks): + raise ValueError,'improper block specification' + + # for non-scalar problems, use pre-defined blocks in aggregation + # the strength of connection matrix is based on the Frobenius norms of the blocks + + B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) + Bt = B.T.tocsr() + + #Frobenius norms of blocks entries of A + #TODO change to 1-norm ? + Block_Frob = Bt * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B + + S = sa_strong_connections(Block_Frob,epsilon) + S.data[:] = 1 + + Mask = B * S * Bt + + A_filtered = A ** Mask + + return A_filtered + + +def sa_strong_connections(A,epsilon,blocks=None): + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') + Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) - #D = diag_sparse(D) - #I,J,V = arange(A.shape[0]).repeat(diff(A.indptr)),A.indices,A.data #COO format for A - #diag_mask = (I == J) - return csr_matrix((Sx,Sj,Sp),A.shape) + def sa_constant_interpolation(A,epsilon,blocks=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - #TODO handle epsilon = 0 case without creating strength of connection matrix? - if blocks is not None: num_dofs = A.shape[0] num_blocks = blocks.max() @@ -35,9 +68,11 @@ # for non-scalar problems, use pre-defined blocks in aggregation # the strength of connection matrix is based on the Frobenius norms of the blocks - + + #TODO change this to matrix 1 norm? B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) - Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B #Frobenius norms of blocks entries of A + #Frobenius norms of blocks entries of A + Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B S = sa_strong_connections(Block_Frob,epsilon) @@ -70,8 +105,8 @@ candidate_matrices = [] for i,c in enumerate(candidates): - #TODO permit incomplete AggOps here (for k-form problems) (other modifications necessary?) - X = csr_matrix((c.copy(),AggOp.indices,AggOp.indptr),dims=AggOp.shape) + c = c[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses + X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) #orthogonalize X against previous @@ -79,7 +114,6 @@ D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X R[j::K,i] = D_AtX X.data -= D_AtX[X.indices] * A.data - #normalize X D_XtX = csr_matrix((X.data**2,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of X.T * X @@ -95,7 +129,7 @@ Q_indices = (K*AggOp.indices).repeat(K) for i in range(K): Q_indices[i::K] += i - Q_data = empty(N_fine * K) + Q_data = empty(AggOp.indptr[-1] * K) #if AggOp includes all nodes, then this is (N_fine * K) for i,X in enumerate(candidate_matrices): Q_data[i::K] = X.data Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) @@ -107,16 +141,17 @@ def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - + AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) - T,coarse_candidates = sa_fit_candidates(AggOp,candidates) + T,coarse_candidates = sa_fit_candidates(AggOp,candidates) - #TODO use filtered matrix here for anisotropic problems - A_filtered = A - D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) + A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems + + D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) D_inv_A = D_inv * A_filtered D_inv_A *= omega/approximate_spectral_radius(D_inv_A) + # smooth tentative prolongator T P = T - (D_inv_A*T) return P,coarse_candidates Modified: trunk/scipy/sandbox/multigrid/simple_test.py =================================================================== --- trunk/scipy/sandbox/multigrid/simple_test.py 2007-10-10 18:06:00 UTC (rev 3432) +++ trunk/scipy/sandbox/multigrid/simple_test.py 2007-10-11 20:53:54 UTC (rev 3433) @@ -1,12 +1,11 @@ from multilevel import * -from multigrid import * from scipy import * A = poisson_problem2D(200) rs_solver = ruge_stuben_solver(A) b = rand(A.shape[0]) -x,res = rs_solver.solve(b,return_residuals=True) -print res +x,residuals = rs_solver.solve(b,return_residuals=True) +print 'residuals',residuals Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-10 18:06:00 UTC (rev 3432) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-11 20:53:54 UTC (rev 3433) @@ -1,7 +1,7 @@ from numpy.testing import * from numpy import sqrt,empty,ones,arange,array_split,eye,array, \ - zeros,diag,zeros_like + zeros,diag,zeros_like,diff from numpy.linalg import norm from scipy import rand from scipy.sparse import spdiags,csr_matrix,lil_matrix, \ @@ -32,175 +32,72 @@ # # return A -def reference_sa_strong_connections(A,epsilon): - A_coo = A.tocoo() - S = lil_matrix(A.shape) +class TestSA(NumpyTestCase): + def setUp(self): + self.cases = [] - for (i,j,v) in zip(A_coo.row,A_coo.col,A_coo.data): - if i == j or abs(v) >= epsilon*sqrt(abs(A[i,i])*abs(A[j,j])): - S[i,j] += v - else: - S[i,i] += v + # random matrices + numpy.random.seed(0) + for N in [2,3,5]: + self.cases.append( csr_matrix(rand(N,N)) ) + + # poisson problems in 1D and 2D + for N in [2,3,5,7,10,11,19]: + self.cases.append( poisson_problem1D(N) ) + for N in [2,3,5,7,10,11]: + self.cases.append( poisson_problem2D(N) ) - return S -class TestSAStrongConnections(NumpyTestCase): -# def check_simple(self): -# N = 4 -# A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() -# assert_array_equal(sa_strong_connections(A,0.50).todense(),A.todense()) #all connections are strong -# assert_array_equal(sa_strong_connections(A,0.51).todense(),0*A.todense()) #no connections are strong -# -# N = 100 -# A = spdiags([2*ones(N),-ones(N),-ones(N)],[0,-1,1],N,N).tocsr() -# assert_array_equal(sa_strong_connections(A,0.50).todense(),A.todense()) #all connections are strong -# assert_array_equal(sa_strong_connections(A,0.51).todense(),0*A.todense()) #no connections are strong - - def check_random(self): - numpy.random.seed(0) - - for N in [2,3,5]: - A = csr_matrix(rand(N,N)) + def check_sa_strong_connections(self): + for A in self.cases: for epsilon in [0.0,0.1,0.5,1.0,10.0]: S_result = sa_strong_connections(A,epsilon) S_expected = reference_sa_strong_connections(A,epsilon) assert_almost_equal(S_result.todense(),S_expected.todense()) #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) - def check_poisson1D(self): - for N in [2,3,5,7,10,11,19]: - A = poisson_problem1D(N) + def check_sa_constant_interpolation(self): + for A in self.cases: for epsilon in [0.0,0.1,0.5,1.0]: - S_result = sa_strong_connections(A,epsilon) - S_expected = reference_sa_strong_connections(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) - - def check_poisson2D(self): - for N in [2,3,5,7,10,11]: - A = poisson_problem2D(N) - for epsilon in [0.0,0.1,0.5,1.0]: - S_result = sa_strong_connections(A,epsilon) - S_expected = reference_sa_strong_connections(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) - - - - -# note that this method only tests the current implementation, not -# all possible implementations -def reference_sa_constant_interpolation(A,epsilon): - S = sa_strong_connections(A,epsilon) - S = array_split(S.indices,S.indptr[1:-1]) - - n = A.shape[0] - - R = set(range(n)) - j = 0 - - aggregates = empty(n,dtype=A.indices.dtype) - aggregates[:] = -1 - - # Pass #1 - for i,row in enumerate(S): - Ni = set(row) | set([i]) - - if Ni.issubset(R): - R -= Ni - for x in Ni: - aggregates[x] = j - j += 1 - - # Pass #2 - Old_R = R.copy() - for i,row in enumerate(S): - if i not in R: continue - - for x in row: - if x not in Old_R: - aggregates[i] = aggregates[x] - R.remove(i) - break - - # Pass #3 - for i,row in enumerate(S): - if i not in R: continue - Ni = set(row) | set([i]) - - for x in Ni: - if x in R: - aggregates[x] = j - j += 1 - - assert(len(R) == 0) - - Pj = aggregates - Pp = arange(n+1) - Px = ones(n) - - return csr_matrix((Px,Pj,Pp)) - -class TestSAConstantInterpolation(NumpyTestCase): - def check_random(self): - numpy.random.seed(0) - for N in [2,3,5,10]: - A = csr_matrix(rand(N,N)) - for epsilon in [0.0,0.1,0.5,1.0]: S_result = sa_constant_interpolation(A,epsilon) S_expected = reference_sa_constant_interpolation(A,epsilon) assert_array_equal(S_result.todense(),S_expected.todense()) - def check_poisson1D(self): - for N in [2,3,5,7,10,11,20,21,29,30]: - A = poisson_problem1D(N) - for epsilon in [0.0,0.1,0.5,1.0]: - S_result = sa_constant_interpolation(A,epsilon) - S_expected = reference_sa_constant_interpolation(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - def check_poisson2D(self): - for N in [2,3,5,7,10,11]: - A = poisson_problem2D(N) - for epsilon in [0.0,0.1,0.5,1.0]: - S_result = sa_constant_interpolation(A,epsilon) - S_expected = reference_sa_constant_interpolation(A,epsilon) - assert_array_equal(S_result.todense(),S_expected.todense()) - -## def check_sample_data(self): -## from examples import all_examples,read_matrix -## -## for filename in all_examples: -## A = read_matrix(filename) -## for epsilon in [0.0,0.08,0.51,1.0]: -## S_result = sa_constant_interpolation(A,epsilon) -## S_expected = reference_sa_constant_interpolation(A,epsilon) -## assert_array_equal((S_result - S_expected).nnz,0) - class TestFitCandidates(NumpyTestCase): def setUp(self): - self.normal_cases = [] + self.cases = [] + ## tests where AggOp includes all DOFs #one candidate - self.normal_cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)),[ones(5)])) - self.normal_cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)),[ones(5)])) - self.normal_cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9)])) - self.normal_cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)),[arange(9)])) - + self.cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)),[ones(5)])) + self.cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)),[ones(5)])) + self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9)])) + self.cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)),[arange(9)])) #two candidates - self.normal_cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)),[ones(4),arange(4)])) - self.normal_cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9),arange(9)])) - self.normal_cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)),[ones(9),arange(9)])) - + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)),[ones(4),arange(4)])) + self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9),arange(9)])) + self.cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)),[ones(9),arange(9)])) #block candidates - self.normal_cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[array([1]*9 + [0]*9),arange(2*9)])) + self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[array([1]*9 + [0]*9),arange(2*9)])) + + ## tests where AggOp excludes some DOFs + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)),[ones(5)])) + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)),[ones(5),arange(5)])) + self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)),[ones(9),arange(9)])) - #TODO add test case where aggregation operator has holes - - def check_normal(self): + def check_all_cases(self): """Test case where aggregation includes all fine nodes""" - for AggOp,fine_candidates in self.normal_cases: + def mask_candidate(AggOp,candidates): + #mask out all DOFs that are not included in the aggregation + for c in candidates: + c[diff(AggOp.indptr) == 0] = 0 + + for AggOp,fine_candidates in self.cases: + + mask_candidate(AggOp,fine_candidates) + Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) assert_equal(len(coarse_candidates),len(fine_candidates)) @@ -210,13 +107,14 @@ assert_almost_equal(fine,Q*coarse) assert_almost_equal(Q*(Q.T*fine),fine) - #aggregate one more level (to a single aggregate) K = len(coarse_candidates) N = K*AggOp.shape[1] AggOp = csr_matrix((ones(N),zeros(N),arange(N + 1)),dims=(N,1)) #aggregate to a single point fine_candidates = coarse_candidates + #mask_candidate(AggOp,fine_candidates) #not needed + #now check the coarser problem Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) @@ -229,15 +127,16 @@ + class TestSASolver(NumpyTestCase): def setUp(self): self.cases = [] - #self.cases.append((poisson_problem1D(10),None)) - - self.cases.append((poisson_problem1D(500),None)) + self.cases.append((poisson_problem1D(100),None)) self.cases.append((poisson_problem2D(50),None)) - + # TODO add unstructured tests + + def check_basic(self): """check that method converges at a reasonable rate""" @@ -256,21 +155,23 @@ assert(avg_convergence_ratio < 0.5) def check_DAD(self): - for A,candidates in self.cases: x = rand(A.shape[0]) - b = A*rand(A.shape[0]) #zeros_like(x) + b = A*rand(A.shape[0]) - D = diag_sparse(rand(A.shape[0])) + D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() D_inv = diag_sparse(1.0/D.data) - DAD = D*A*D + + DAD = D*A*D if candidates is None: candidates = [ ones(A.shape[0]) ] DAD_candidates = [ (D_inv * c) for c in candidates ] - + + #TODO force 2 level method and check that result is the same + #ml = smoothed_aggregation_solver(A,candidates,max_coarse=1,max_levels=2) ml = smoothed_aggregation_solver(DAD,DAD_candidates,max_coarse=100,max_levels=2) @@ -280,38 +181,84 @@ x_sol,residuals = ml.solve(b,x0=x,maxiter=10,tol=1e-12,return_residuals=True) avg_convergence_ratio = (residuals[-1]/residuals[0])**(1.0/len(residuals)) - print avg_convergence_ratio + #print avg_convergence_ratio assert(avg_convergence_ratio < 0.5) + + + + +################################################ +## reference implementations for unittests ## +################################################ +def reference_sa_strong_connections(A,epsilon): + A_coo = A.tocoo() + S = lil_matrix(A.shape) + + for (i,j,v) in zip(A_coo.row,A_coo.col,A_coo.data): + if i == j or abs(v) >= epsilon*sqrt(abs(A[i,i])*abs(A[j,j])): + S[i,j] += v + else: + S[i,i] += v + + return S + + +# note that this method only tests the current implementation, not +# all possible implementations +def reference_sa_constant_interpolation(A,epsilon): + S = sa_strong_connections(A,epsilon) + S = array_split(S.indices,S.indptr[1:-1]) + + n = A.shape[0] + + R = set(range(n)) + j = 0 + + aggregates = empty(n,dtype=A.indices.dtype) + aggregates[:] = -1 + + # Pass #1 + for i,row in enumerate(S): + Ni = set(row) | set([i]) + + if Ni.issubset(R): + R -= Ni + for x in Ni: + aggregates[x] = j + j += 1 + + # Pass #2 + Old_R = R.copy() + for i,row in enumerate(S): + if i not in R: continue -## def check_DAD(self): -## """check that method is invariant to symmetric diagonal scaling (A -> DAD)""" -## -## for A,A_candidates in self.cases: -## numpy.random.seed(0) #make tests repeatable -## -## x = rand(A.shape[0]) -## b = zeros_like(x) -## -## D = diag_sparse(rand(A.shape[0])) -## D_inv = diag_sparse(1.0/D.data) -## DAD = D*A*D -## -## -## if A_candidates is None: -## A_candidates = [ ones(A.shape[0]) ] -## -## DAD_candidates = [ (D_inv * c) for c in A_candidates ] -## -## ml_A = smoothed_aggregation_solver(A, A_candidates, max_coarse=10, max_levels=10, epsilon=0.0) -## x_sol_A = ml_A.solve(b, x0=x, maxiter=1, tol=1e-12) -## -## ml_DAD = smoothed_aggregation_solver(DAD, DAD_candidates, max_coarse=10, max_levels=10, epsilon=0.0) -## x_sol_DAD = ml_DAD.solve(b, x0=D*x, maxiter=1, tol=1e-12) -## -## assert_almost_equal(x_sol_A, D_inv * x_sol_DAD) + for x in row: + if x not in Old_R: + aggregates[i] = aggregates[x] + R.remove(i) + break + # Pass #3 + for i,row in enumerate(S): + if i not in R: continue + Ni = set(row) | set([i]) + for x in Ni: + if x in R: + aggregates[x] = j + j += 1 + + assert(len(R) == 0) + + Pj = aggregates + Pp = arange(n+1) + Px = ones(n) + + return csr_matrix((Px,Pj,Pp)) + + + if __name__ == '__main__': NumpyTest().run() Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-10 18:06:00 UTC (rev 3432) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-11 20:53:54 UTC (rev 3433) @@ -1,4 +1,5 @@ -__all__ =['approximate_spectral_radius','infinity_norm','diag_sparse'] +__all__ =['approximate_spectral_radius','infinity_norm','diag_sparse', + 'hstack_csr','vstack_csr'] import numpy import scipy @@ -45,3 +46,23 @@ return csr_matrix((A,arange(len(A)),arange(len(A)+1)),(len(A),len(A))) +def hstack_csr(A,B): + #TODO OPTIMIZE THIS + assert(A.shape[0] == B.shape[0]) + A = A.tocoo() + B = B.tocoo() + I = concatenate((A.row,B.row)) + J = concatenate((A.col,B.col+A.shape[1])) + V = concatenate((A.data,B.data)) + return coo_matrix((V,(I,J)),dims=(A.shape[0],A.shape[1]+B.shape[1])).tocsr() + +def vstack_csr(A,B): + #TODO OPTIMIZE THIS + assert(A.shape[1] == B.shape[1]) + A = A.tocoo() + B = B.tocoo() + I = concatenate((A.row,B.row+A.shape[0])) + J = concatenate((A.col,B.col)) + V = concatenate((A.data,B.data)) + return coo_matrix((V,(I,J)),dims=(A.shape[0]+B.shape[0],A.shape[1])).tocsr() + From scipy-svn at scipy.org Fri Oct 12 13:17:00 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 12 Oct 2007 12:17:00 -0500 (CDT) Subject: [Scipy-svn] r3434 - trunk/scipy/stats Message-ID: <20071012171700.4F88D39C050@new.scipy.org> Author: rkern Date: 2007-10-12 12:16:54 -0500 (Fri, 12 Oct 2007) New Revision: 3434 Modified: trunk/scipy/stats/kde.py Log: Fix typo. Modified: trunk/scipy/stats/kde.py =================================================================== --- trunk/scipy/stats/kde.py 2007-10-11 20:53:54 UTC (rev 3433) +++ trunk/scipy/stats/kde.py 2007-10-12 17:16:54 UTC (rev 3434) @@ -202,7 +202,7 @@ if self.d != 1: raise ValueError("integrate_box_1d() only handles 1D pdfs") - stdev = np.ravel(sqrt(self.covariance))[0] + stdev = ravel(sqrt(self.covariance))[0] normalized_low = ravel((low - self.dataset)/stdev) normalized_high = ravel((high - self.dataset)/stdev) From scipy-svn at scipy.org Sat Oct 13 14:33:37 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sat, 13 Oct 2007 13:33:37 -0500 (CDT) Subject: [Scipy-svn] r3435 - in trunk/scipy/io: . tests Message-ID: <20071013183337.9860B39C089@new.scipy.org> Author: matthew.brett at gmail.com Date: 2007-10-13 13:33:22 -0500 (Sat, 13 Oct 2007) New Revision: 3435 Modified: trunk/scipy/io/mio.py trunk/scipy/io/mio5.py trunk/scipy/io/tests/test_mio.py Log: Basic matlab 5 support - thanks to Lars Voxen Hansen Modified: trunk/scipy/io/mio.py =================================================================== --- trunk/scipy/io/mio.py 2007-10-12 17:16:54 UTC (rev 3434) +++ trunk/scipy/io/mio.py 2007-10-13 18:33:22 UTC (rev 3435) @@ -27,7 +27,7 @@ else: full_name = None junk, file_name = os.path.split(file_name) - for path in sys.path: + for path in [os.curdir] + list(sys.path): test_name = os.path.join(path, file_name) if appendmat: test_name += ".mat" @@ -100,13 +100,14 @@ mdict = matfile_dict return mdict -def savemat(file_name, mdict, appendmat=True): +def savemat(file_name, mdict, appendmat=True, format='4'): """Save a dictionary of names and arrays into the MATLAB-style .mat file. This saves the arrayobjects in the given dictionary to a matlab - Version 4 style .mat file. + style .mat file. - @appendmat - if true, appends '.mat' extension to filename, if not present + appendmat - if true, appends '.mat' extension to filename, if not present + format - '4' for matlab 4 mat files, '5' for matlab 5 onwards """ file_is_string = isinstance(file_name, basestring) if file_is_string: @@ -121,7 +122,12 @@ 'file-like object' file_stream = file_name - MW = MatFile4Writer(file_stream) + if format == '4': + MW = MatFile4Writer(file_stream) + elif format == '5': + MW = MatFile5Writer(file_stream) + else: + raise ValueError, 'Format should be 4 or 5' MW.put_variables(mdict) if file_is_string: file_stream.close() Modified: trunk/scipy/io/mio5.py =================================================================== --- trunk/scipy/io/mio5.py 2007-10-12 17:16:54 UTC (rev 3434) +++ trunk/scipy/io/mio5.py 2007-10-13 18:33:22 UTC (rev 3435) @@ -105,6 +105,41 @@ mxDOUBLE_CLASS: 'f8', } + +np_to_mtypes = { + 'f8': miDOUBLE, + 'c32': miDOUBLE, + 'c24': miDOUBLE, + 'c16': miDOUBLE, + 'f4': miSINGLE, + 'c8': miSINGLE, + 'i1': miINT8, + 'i2': miINT16, + 'i4': miINT32, + 'u1': miUINT8, + 'u4': miUINT32, + 'u2': miUINT16, + 'S1': miUINT8, + 'U1': miUTF16, + } + + +np_to_mxtypes = { + 'f8': mxDOUBLE_CLASS, + 'c32': mxDOUBLE_CLASS, + 'c24': mxDOUBLE_CLASS, + 'c16': mxDOUBLE_CLASS, + 'f4': mxSINGLE_CLASS, + 'c8': mxSINGLE_CLASS, + 'i4': mxINT32_CLASS, + 'i2': mxINT16_CLASS, + 'u2': mxUINT16_CLASS, + 'u1': mxUINT8_CLASS, + 'S1': mxUINT8_CLASS, + } + + + ''' Before release v7.1 (release 14) matlab (TM) used the system default character encoding scheme padded out to 16-bits. Release 14 and later use Unicode. When saving character data, R14 checks if it @@ -532,12 +567,22 @@ self.is_global = is_global def write_dtype(self, arr): - self.file_stream.write(arr.tostring) + self.file_stream.write(arr.tostring()) - def write_element(self, arr): - # check if small element works - do it + def write_element(self, arr, mdtype=None): # write tag, data - pass + tag = N.zeros((), mdtypes_template['tag_full']) + if mdtype is None: + tag['mdtype'] = np_to_mtypes[arr.dtype.str[1:]] + else: + tag['mdtype'] = mdtype + tag['byte_count'] = arr.size*arr.itemsize + self.write_dtype(tag) + self.write_bytes(arr) + # do 8 byte padding if needed + if tag['byte_count']%8 != 0: + pad = (1+tag['byte_count']//8)*8 - tag['byte_count'] + self.write_bytes(N.zeros((pad,),dtype='u1')) def write_header(self, mclass, is_global=False, @@ -561,8 +606,11 @@ af['flags_class'] = mclass | flags << 8 af['nzmax'] = nzmax self.write_dtype(af) + # write array shape + self.arr=N.atleast_2d(self.arr) self.write_element(N.array(self.arr.shape, dtype='i4')) - self.write_element(self.name) + # write name + self.write_element(N.ndarray(shape=len(self.name), dtype='S1', buffer=self.name)) def update_matrix_tag(self): curr_pos = self.file_stream.tell() @@ -578,34 +626,43 @@ class Mat5NumericWriter(Mat5MatrixWriter): def write(self): - # identify matlab type for array - # make at least 2d - # maybe downcast array to smaller matlab type - # write real - # write imaginary - # put padded length in miMATRIX tag - pass - + imagf = self.arr.dtype.kind == 'c' + try: + mclass = np_to_mxtypes[self.arr.dtype.str[1:]] + except KeyError: + if imagf: + self.arr = self.arr.astype('c128') + else: + self.arr = self.arr.astype('f8') + mclass = mxDOUBLE_CLASS + self.write_header(mclass=mclass,is_complex=imagf) + if imagf: + self.write_element(self.arr.real) + self.write_element(self.arr.imag) + else: + self.write_element(self.arr) + self.update_matrix_tag() class Mat5CharWriter(Mat5MatrixWriter): - + codec='ascii' def write(self): self.arr_to_chars() - self.arr_to_2d() - dims = self.arr.shape - self.write_header(P=miUINT8, - T=mxCHAR_CLASS) + self.write_header(mclass=mxCHAR_CLASS) if self.arr.dtype.kind == 'U': - # Recode unicode to ascii - n_chars = N.product(dims) + # Recode unicode using self.codec + n_chars = N.product(self.arr.shape) st_arr = N.ndarray(shape=(), dtype=self.arr_dtype_number(n_chars), buffer=self.arr) - st = st_arr.item().encode('ascii') - self.arr = N.ndarray(shape=dims, dtype='S1', buffer=st) - self.write_bytes(self.arr) + st = st_arr.item().encode(self.codec) + self.arr = N.ndarray(shape=(len(st)), dtype='u1', buffer=st) + self.write_element(self.arr,mdtype=miUTF8) + self.update_matrix_tag() +class Mat5UniCharWriter(Mat5CharWriter): + codec='UTF8' + class Mat5SparseWriter(Mat5MatrixWriter): def write(self): @@ -650,7 +707,7 @@ return Mat5SparseWriter(self.stream, arr, name, is_global) arr = N.array(arr) if arr.dtype.hasobject: - types, arr_type = classify_mobjects(arr) + types, arr_type = self.classify_mobjects(arr) if arr_type == 'c': return Mat5CellWriter(self.stream, arr, name, is_global, types) elif arr_type == 's': @@ -658,10 +715,10 @@ elif arr_type == 'o': return Mat5ObjectWriter(self.stream, arr, name, is_global) if arr.dtype.kind in ('U', 'S'): - if self.unicode_strings: + if self.unicode_strings: return Mat5UniCharWriter(self.stream, arr, name, is_global) - else: - return Mat5IntCharWriter(self.stream, arr, name, is_global) + else: + return Mat5CharWriter(self.stream, arr, name, is_global) else: return Mat5NumericWriter(self.stream, arr, name, is_global) @@ -678,12 +735,12 @@ s - struct array o - object array ''' - N = objarr.size - types = N.empty((N,), dtype='S1') + n = objarr.size + types = N.empty((n,), dtype='S1') types[:] = 'i' type_set = set() flato = objarr.flat - for i in range(N): + for i in range(n): obj = flato[i] if isinstance(obj, N.ndarray): types[i] = 'a' @@ -719,8 +776,16 @@ else: self.global_vars = [] self.writer_getter = Mat5WriterGetter( - StringIO, + StringIO(), unicode_strings) + # write header + import os, time + hdr = N.zeros((), mdtypes_template['file_header']) + hdr['description']='MATLAB 5.0 MAT-file Platform: %s, Created on: %s' % ( + os.name,time.asctime()) + hdr['version']= 0x0100 + hdr['endian_test']=N.ndarray(shape=(),dtype='S2',buffer=N.uint16(0x4d49)) + file_stream.write(hdr.tostring()) def get_unicode_strings(self): return self.write_getter.unicode_strings @@ -740,11 +805,12 @@ name, is_global, ).write() + stream = self.writer_getter.stream if self.do_compression: - str = zlib.compress(stream.getvalue()) + str = zlib.compress(stream.getvalue(stream.tell())) tag = N.empty((), mdtypes_template['tag_full']) tag['mdtype'] = miCOMPRESSED tag['byte_count'] = len(str) self.file_stream.write(tag.tostring() + str) else: - self.file_stream.write(stream.getvalue()) + self.file_stream.write(stream.getvalue(stream.tell())) Modified: trunk/scipy/io/tests/test_mio.py =================================================================== --- trunk/scipy/io/tests/test_mio.py 2007-10-12 17:16:54 UTC (rev 3434) +++ trunk/scipy/io/tests/test_mio.py 2007-10-13 18:33:22 UTC (rev 3435) @@ -13,7 +13,6 @@ set_package_path() from scipy.io.mio import loadmat, savemat from scipy.io.mio5 import mat_obj, mat_struct -from scipy.io.mio4 import MatFile4Writer restore_path() try: # Python 2.3 support @@ -87,10 +86,10 @@ return cc # Add the round trip tests dynamically, with given parameters - def _make_rt_check_case(name, expected): + def _make_rt_check_case(name, expected, format): def cc(self): mat_stream = StringIO() - savemat(mat_stream, expected) + savemat(mat_stream, expected, format) mat_stream.seek(0) self._check_case(name, [mat_stream], expected) cc.__doc__ = "check loadmat case %s" % name @@ -226,10 +225,12 @@ assert files, "No files for test %s using filter %s" % (name, filt) exec 'check_%s = _make_check_case(name, files, expected)' % name # round trip tests - for case in case_table4: + for case in case_table4 + case_table5: name = case['name'] + '_round_trip' expected = case['expected'] - exec 'check_%s = _make_rt_check_case(name, expected)' % name + format = case in case_table4 and '4' or '5' + exec 'check_%s = _make_rt_check_case(name, expected, format)' \ + % name if __name__ == "__main__": From scipy-svn at scipy.org Mon Oct 15 15:23:10 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 15 Oct 2007 14:23:10 -0500 (CDT) Subject: [Scipy-svn] r3436 - in trunk/scipy/sparse: . tests Message-ID: <20071015192310.BDC8539C239@new.scipy.org> Author: wnbell Date: 2007-10-15 14:23:07 -0500 (Mon, 15 Oct 2007) New Revision: 3436 Modified: trunk/scipy/sparse/sparse.py trunk/scipy/sparse/tests/test_sparse.py Log: fix sparse matvec result dimensions check matvec input dimensions resolves ticket #514 Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-13 18:33:22 UTC (rev 3435) +++ trunk/scipy/sparse/sparse.py 2007-10-15 19:23:07 UTC (rev 3436) @@ -673,23 +673,25 @@ def _matvec(self, other, fn): if isdense(other): - # This check is too harsh -- it prevents a column vector from - # being created on-the-fly like dense matrix objects can. - #if len(other) != self.shape[1]: - # raise ValueError, "dimension mismatch" - oth = numpy.ravel(other) + if other.size != self.shape[1] or \ + (other.ndim == 2 and self.shape[1] != other.shape[0]): + raise ValueError, "dimension mismatch" + y = fn(self.shape[0], self.shape[1], \ - self.indptr, self.indices, self.data, oth) + self.indptr, self.indices, self.data, numpy.ravel(other)) + if isinstance(other, matrix): y = asmatrix(y) + if other.ndim == 2 and other.shape[1] == 1: - # If 'other' was an (nx1) column vector, transpose the result - # to obtain an (mx1) column vector. - y = y.T + # If 'other' was an (nx1) column vector, reshape the result + y = y.reshape(-1,1) + return y elif isspmatrix(other): raise TypeError, "use matmat() for sparse * sparse" + else: raise TypeError, "need a dense vector" Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-13 18:33:22 UTC (rev 3435) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-15 19:23:07 UTC (rev 3436) @@ -176,7 +176,23 @@ M = self.spmatrix(matrix([[3,0,0],[0,1,0],[2,0,3.0],[2,3,0]])) col = matrix([1,2,3]).T assert_array_almost_equal(M * col, M.todense() * col) + + #check result dimensions (ticket #514) + assert_equal((M * array([1,2,3])).shape,(4,)) + assert_equal((M * array([[1],[2],[3]])).shape,(4,1)) + assert_equal((M * matrix([[1],[2],[3]])).shape,(4,1)) + #ensure exception is raised for improper dimensions + bad_vecs = [array([1,2]), array([1,2,3,4]), array([[1],[2]]), + matrix([1,2,3]), matrix([[1],[2]])] + caught = 0 + for x in bad_vecs: + try: + y = M * x + except ValueError: + caught += 1 + assert_equal(caught,len(bad_vecs)) + # Should this be supported or not?! #flat = array([1,2,3]) #assert_array_almost_equal(M*flat, M.todense()*flat) From scipy-svn at scipy.org Mon Oct 15 16:33:26 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 15 Oct 2007 15:33:26 -0500 (CDT) Subject: [Scipy-svn] r3437 - trunk/scipy/sandbox/multigrid Message-ID: <20071015203326.A1CEE39C033@new.scipy.org> Author: wnbell Date: 2007-10-15 15:33:20 -0500 (Mon, 15 Oct 2007) New Revision: 3437 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/sa.py Log: make SA work properly w/ blocks Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-15 19:23:07 UTC (rev 3436) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-15 20:33:20 UTC (rev 3437) @@ -16,14 +16,14 @@ """ - #candidate prolongator (assumes every value from x is used) + #candidate prolongator (assumes every value from x is used) #TODO permit gaps X = csr_matrix((x_l,W_l.indices,W_l.indptr),dims=W_l.shape,check=False) R = (P_l.T.tocsr() * X) # R has at most 1 nz per row X = X - P_l*R # othogonalize X against P_l - #DROP REDUNDANT COLUMNS FROM P (AND R?) HERE (NULL OUT R ACCORDINGLY?) - #REMOVE CORRESPONDING COLUMNS FROM W_l AND ROWS FROM A_m ALSO + #TODO DROP REDUNDANT COLUMNS FROM P (AND R?) HERE (NULL OUT R ACCORDINGLY?) + #TODO REMOVE CORRESPONDING COLUMNS FROM W_l AND ROWS FROM A_m ALSO W_l_new = W_l W_m_new = W_m @@ -59,7 +59,7 @@ D = diag_sparse(A) D_inv_A = diag_sparse(1.0/D)*A omega = 4.0/(3.0*approximate_spectral_radius(D_inv_A)) - print "spectral radius",approximate_spectral_radius(D_inv_A) + print "spectral radius",approximate_spectral_radius(D_inv_A) #TODO remove this D_inv_A *= omega return P - D_inv_A*P @@ -124,8 +124,9 @@ x = self.__develop_candidate(A,As,Is,Ps,Ws,AggOps,mu=mu) self.candidates.append(x) - - #As,Is,Ps,Ws = self.__augment_cycle(A,As,Ps,Ws,AggOps,x) + + #TODO which is faster? + #As,Is,Ps,Ws = self.__augment_cycle(A,As,Ps,Ws,AggOps,x) As,Is,Ps = sa_hierarchy(A,AggOps,self.candidates) self.Ps = Ps Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-15 19:23:07 UTC (rev 3436) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-15 20:33:20 UTC (rev 3437) @@ -22,7 +22,7 @@ """ D = 2*ones(N) O = -ones(N) - return scipy.sparse.spdiags([D,O,O],[0,-1,1],N,N).tocoo().tocsr() #eliminate zeros + return scipy.sparse.spdiags([D,O,O],[0,-1,1],N,N).tocoo().tocsr() #eliminate explicit zeros def poisson_problem2D(N): """ @@ -34,7 +34,7 @@ T = -ones(N*N) O = -ones(N*N) T[N-1::N] = 0 - return scipy.sparse.spdiags([D,O,T,T,O],[0,-N,-1,1,N],N*N,N*N).tocoo().tocsr() #eliminate zeros + return scipy.sparse.spdiags([D,O,T,T,O],[0,-N,-1,1,N],N*N,N*N).tocoo().tocsr() #eliminate explicit zeros def ruge_stuben_solver(A,max_levels=10,max_coarse=500): @@ -60,7 +60,7 @@ return multilevel_solver(As,Ps) -def smoothed_aggregation_solver(A,candidates=None,blocks=None,max_levels=10,max_coarse=500,epsilon=0.08,omega=4.0/3.0): +def smoothed_aggregation_solver(A,candidates=None,blocks=None,aggregation=None,max_levels=10,max_coarse=500,epsilon=0.08,omega=4.0/3.0): """ Create a multilevel solver using Smoothed Aggregation (SA) @@ -75,15 +75,25 @@ if candidates is None: candidates = [ ones(A.shape[0]) ] # use constant vector - - while len(As) < max_levels and A.shape[0] > max_coarse: - P,candidates = sa_interpolation(A,candidates,epsilon*0.5**(len(As)-1),omega=omega,blocks=blocks) - #P = sa_interpolation(A,epsilon=0.0) - A = (P.T.tocsr() * A) * P #galerkin operator + if aggregation is None: + while len(As) < max_levels and A.shape[0] > max_coarse: + P,candidates = sa_interpolation(A,candidates,epsilon*0.5**(len(As)-1),omega=omega,blocks=blocks) + blocks = None #only used for 1st level - As.append(A) - Ps.append(P) + A = (P.T.tocsr() * A) * P #galerkin operator + + As.append(A) + Ps.append(P) + else: + #use user-defined aggregation + for AggOp in aggregation: + P,candidates = sa_interpolation(A,candidates,omega=omega,AggOp=AggOp) + + A = (P.T.tocsr() * A) * P #galerkin operator + + As.append(A) + Ps.append(P) return multilevel_solver(As,Ps) @@ -163,7 +173,7 @@ if lvl == len(self.As) - 2: #use direct solver on coarsest level - coarse_x[:] = spsolve(self.As[-1],coarse_b) + coarse_x[:] = spsolve(self.As[-1],coarse_b) #TODO reuse factors for efficiency? #coarse_x[:] = scipy.linalg.cg(self.As[-1],coarse_b,tol=1e-12)[0] #print "coarse residual norm",scipy.linalg.norm(coarse_b - self.As[-1]*coarse_x) else: @@ -185,17 +195,19 @@ if __name__ == '__main__': from scipy import * candidates = None - A = poisson_problem2D(100) + blocks = None + #A = poisson_problem2D(100) #A = io.mmread("rocker_arm_surface.mtx").tocsr() #A = io.mmread("9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() - #A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() - #candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') - #candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] - - ml = smoothed_aggregation_solver(A,candidates,max_coarse=10,max_levels=10) + A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() + candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') + candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] + blocks = arange(A.shape[0]/2).repeat(2) + + ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,max_coarse=10,max_levels=10) #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) @@ -210,8 +222,12 @@ residuals.append(linalg.norm(b - A*x)) A.psolve = ml.psolve x_sol = linalg.cg(A,b,x0=x,maxiter=12,tol=1e-100,callback=add_resid)[0] + residuals = array(residuals)/residuals[0] + avg_convergence_ratio = residuals[-1]**(1.0/len(residuals)) + print "average convergence ratio",avg_convergence_ratio + print "last convergence ratio",residuals[-1]/residuals[-2] print residuals Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-15 19:23:07 UTC (rev 3436) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-15 20:33:20 UTC (rev 3437) @@ -1,6 +1,7 @@ import scipy import numpy -from numpy import array,arange,ones,zeros,sqrt,isinf,asarray,empty,diff +from numpy import array,arange,ones,zeros,sqrt,isinf,asarray,empty,diff,\ + ascontiguousarray from scipy.sparse import csr_matrix,isspmatrix_csr from utils import diag_sparse,approximate_spectral_radius @@ -23,7 +24,7 @@ else: num_dofs = A.shape[0] - num_blocks = blocks.max() + num_blocks = blocks.max() + 1 if num_dofs != len(blocks): raise ValueError,'improper block specification' @@ -34,11 +35,10 @@ B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) Bt = B.T.tocsr() - #Frobenius norms of blocks entries of A - #TODO change to 1-norm ? - Block_Frob = Bt * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B + #1-norms of blocks entries of A + Block_A = Bt * csr_matrix((abs(A.data),A.indices,A.indptr),dims=A.shape) * B - S = sa_strong_connections(Block_Frob,epsilon) + S = sa_strong_connections(Block_A,epsilon) S.data[:] = 1 Mask = B * S * Bt @@ -61,20 +61,20 @@ if blocks is not None: num_dofs = A.shape[0] - num_blocks = blocks.max() + num_blocks = blocks.max() + 1 if num_dofs != len(blocks): raise ValueError,'improper block specification' - + + print "SA has blocks" # for non-scalar problems, use pre-defined blocks in aggregation # the strength of connection matrix is based on the Frobenius norms of the blocks - #TODO change this to matrix 1 norm? B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) - #Frobenius norms of blocks entries of A - Block_Frob = B.T.tocsr() * csr_matrix((A.data**2,A.indices,A.indptr),dims=A.shape) * B + #1-norms of blocks entries of A + Block_A = B.T.tocsr() * csr_matrix((abs(A.data),A.indices,A.indptr),dims=A.shape) * B - S = sa_strong_connections(Block_Frob,epsilon) + S = sa_strong_connections(Block_A,epsilon) Pj = multigridtools.sa_get_aggregates(S.shape[0],S.indptr,S.indices) Pj = Pj[blocks] #expand block aggregates into constituent dofs @@ -108,7 +108,6 @@ c = c[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) - #orthogonalize X against previous for j,A in enumerate(candidate_matrices): D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X @@ -134,15 +133,22 @@ Q_data[i::K] = X.data Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) - coarse_candidates = [array(R[:,i]) for i in range(K)] + coarse_candidates = [ascontiguousarray(R[:,i]) for i in range(K)] return Q,coarse_candidates -def sa_interpolation(A,candidates,epsilon,omega=4.0/3.0,blocks=None): +def sa_interpolation(A,candidates,epsilon=0.0,omega=4.0/3.0,blocks=None,AggOp=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') - AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) + if AggOp is None: + AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) + else: + if not isspmatrix_csr(AggOp): + raise TypeError,'aggregation is specified by a list of csr_matrix objects' + if A.shape[1] != AggOp.shape[0]: + raise ValueError,'incompatible aggregation operator' + T,coarse_candidates = sa_fit_candidates(AggOp,candidates) A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems From scipy-svn at scipy.org Mon Oct 15 16:43:49 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 15 Oct 2007 15:43:49 -0500 (CDT) Subject: [Scipy-svn] r3438 - trunk/scipy/sandbox/maskedarray Message-ID: <20071015204349.313FD39C238@new.scipy.org> Author: pierregm Date: 2007-10-15 15:43:47 -0500 (Mon, 15 Oct 2007) New Revision: 3438 Modified: trunk/scipy/sandbox/maskedarray/core.py Log: fixed putmask. Thx to Eric Firing for finding and correcting the bug. Modified: trunk/scipy/sandbox/maskedarray/core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/core.py 2007-10-15 20:33:20 UTC (rev 3437) +++ trunk/scipy/sandbox/maskedarray/core.py 2007-10-15 20:43:47 UTC (rev 3438) @@ -2588,7 +2588,7 @@ try: return a.putmask(values, mask) except AttributeError: - return narray(a, copy=False).putmask(values, mask) + return numpy.putmask(narray(a, copy=False), mask, values) def transpose(a,axes=None): """Returns a view of the array with dimensions permuted according to axes, From scipy-svn at scipy.org Mon Oct 15 18:47:00 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 15 Oct 2007 17:47:00 -0500 (CDT) Subject: [Scipy-svn] r3439 - trunk/scipy/sparse Message-ID: <20071015224700.61D4339C19B@new.scipy.org> Author: nmarais Date: 2007-10-15 17:46:49 -0500 (Mon, 15 Oct 2007) New Revision: 3439 Modified: trunk/scipy/sparse/sparse.py Log: Fix sparse.listprint() to avoid repeated string concatenation Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-15 20:43:47 UTC (rev 3438) +++ trunk/scipy/sparse/sparse.py 2007-10-15 22:46:49 UTC (rev 3439) @@ -175,10 +175,8 @@ def listprint(self, start, stop): """Provides a way to print over a single index. """ - val = '' - for ind in xrange(start, stop): - val = val + ' %s\t%s\n' % (self.rowcol(ind), self.getdata(ind)) - return val + return '\n'.join(' %s\t%s' % (self.rowcol(ind), self.getdata(ind)) + for ind in xrange(start,stop)) + '\n' def __repr__(self): nnz = self.getnnz() From scipy-svn at scipy.org Tue Oct 16 02:19:15 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 16 Oct 2007 01:19:15 -0500 (CDT) Subject: [Scipy-svn] r3440 - in trunk/scipy/sandbox/multigrid: . tests Message-ID: <20071016061915.B04ED39C0BB@new.scipy.org> Author: wnbell Date: 2007-10-16 01:19:12 -0500 (Tue, 16 Oct 2007) New Revision: 3440 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/tests/test_sa.py trunk/scipy/sandbox/multigrid/utils.py Log: add block option to adaptive SA Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-15 22:46:49 UTC (rev 3439) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-16 06:19:12 UTC (rev 3440) @@ -105,13 +105,13 @@ self.Rs = [] - if self.A.shape[0] <= self.opts['coarse: max size']: + if self.A.shape[0] <= max_coarse: raise ValueError,'small matrices not handled yet' #first candidate - x,AggOps = self.__initialization_stage(A,blocks=blocks,\ - max_levels=max_levels,max_coarse=max_coarse,\ - mu=mu,epsilon=epsilon) + x,AggOps = self.__initialization_stage(A, blocks=blocks,\ + max_levels=max_levels, max_coarse=max_coarse,\ + mu=mu, epsilon=epsilon) Ws = AggOps @@ -134,7 +134,7 @@ self.AggOps = AggOps - def __initialization_stage(self,A,max_levels,max_coarse,mu,epsilon): + def __initialization_stage(self,A,blocks,max_levels,max_coarse,mu,epsilon): AggOps = [] Is = [] @@ -270,6 +270,9 @@ from scipy import * from utils import diag_sparse from multilevel import poisson_problem1D,poisson_problem2D + +blocks = None + A = poisson_problem2D(200) #A = io.mmread("tests/sample_data/laplacian_41_3dcube.mtx").tocsr() #A = io.mmread("laplacian_40_3dcube.mtx").tocsr() @@ -280,16 +283,17 @@ #D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() #A = D * A * D -#A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() +A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() +blocks = arange(A.shape[0]/2).repeat(2) -asa = adaptive_sa_solver(A,max_candidates=1,mu=5) +asa = adaptive_sa_solver(A,max_candidates=4,mu=12) scipy.random.seed(0) #make tests repeatable x = rand(A.shape[0]) b = A*rand(A.shape[0]) print "solving" -if False: +if True: x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=10,tol=1e-12,return_residuals=True) else: residuals = [] Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-15 22:46:49 UTC (rev 3439) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-16 06:19:12 UTC (rev 3440) @@ -21,34 +21,37 @@ if blocks is None: Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) A_filtered = csr_matrix((Sx,Sj,Sp),A.shape) - else: - num_dofs = A.shape[0] - num_blocks = blocks.max() + 1 - - if num_dofs != len(blocks): - raise ValueError,'improper block specification' - - # for non-scalar problems, use pre-defined blocks in aggregation - # the strength of connection matrix is based on the Frobenius norms of the blocks - - B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) - Bt = B.T.tocsr() - - #1-norms of blocks entries of A - Block_A = Bt * csr_matrix((abs(A.data),A.indices,A.indptr),dims=A.shape) * B - - S = sa_strong_connections(Block_A,epsilon) - S.data[:] = 1 + A_filtered = A #TODO subtract weak blocks from diagonal blocks? - Mask = B * S * Bt +## num_dofs = A.shape[0] +## num_blocks = blocks.max() + 1 +## +## if num_dofs != len(blocks): +## raise ValueError,'improper block specification' +## +## # for non-scalar problems, use pre-defined blocks in aggregation +## # the strength of connection matrix is based on the 1-norms of the blocks +## +## B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) +## Bt = B.T.tocsr() +## +## #1-norms of blocks entries of A +## Block_A = Bt * csr_matrix((abs(A.data),A.indices,A.indptr),dims=A.shape) * B +## +## S = sa_strong_connections(Block_A,epsilon) +## S.data[:] = 1 +## +## Mask = B * S * Bt +## +## A_strong = A ** Mask +## #A_weak = A - A_strong +## A_filtered = A_strong - A_filtered = A ** Mask - return A_filtered -def sa_strong_connections(A,epsilon,blocks=None): +def sa_strong_connections(A,epsilon): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) @@ -66,7 +69,7 @@ if num_dofs != len(blocks): raise ValueError,'improper block specification' - print "SA has blocks" + #print "SA has blocks" # for non-scalar problems, use pre-defined blocks in aggregation # the strength of connection matrix is based on the Frobenius norms of the blocks Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-15 22:46:49 UTC (rev 3439) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-16 06:19:12 UTC (rev 3440) @@ -1,7 +1,7 @@ from numpy.testing import * from numpy import sqrt,empty,ones,arange,array_split,eye,array, \ - zeros,diag,zeros_like,diff + zeros,diag,zeros_like,diff,matrix from numpy.linalg import norm from scipy import rand from scipy.sparse import spdiags,csr_matrix,lil_matrix, \ @@ -51,19 +51,42 @@ def check_sa_strong_connections(self): for A in self.cases: for epsilon in [0.0,0.1,0.5,1.0,10.0]: + S_expected = reference_sa_strong_connections(A,epsilon) S_result = sa_strong_connections(A,epsilon) - S_expected = reference_sa_strong_connections(A,epsilon) assert_almost_equal(S_result.todense(),S_expected.todense()) #assert_array_equal(sparsity(S_result).todense(),sparsity(S_expected).todense()) def check_sa_constant_interpolation(self): for A in self.cases: for epsilon in [0.0,0.1,0.5,1.0]: - S_result = sa_constant_interpolation(A,epsilon) S_expected = reference_sa_constant_interpolation(A,epsilon) + + S_result = sa_constant_interpolation(A,epsilon,blocks=None) assert_array_equal(S_result.todense(),S_expected.todense()) + + #blocks=1...N should be the same as blocks=None + S_result = sa_constant_interpolation(A,epsilon,blocks=arange(A.shape[0])) + assert_array_equal(S_result.todense(),S_expected.todense()) + #check simple block examples + A = csr_matrix(arange(16).reshape(4,4)) + A = A + A.T + blocks = array([0,0,1,1]) + S_result = sa_constant_interpolation(A,epsilon=0.0,blocks=blocks) + S_expected = matrix([1,1,1,1]).T + assert_array_equal(S_result.todense(),S_expected) + + S_result = sa_constant_interpolation(A,epsilon=0.5,blocks=blocks) + S_expected = matrix([1,1,1,1]).T + assert_array_equal(S_result.todense(),S_expected) + + S_result = sa_constant_interpolation(A,epsilon=2.0,blocks=blocks) + S_expected = matrix([[1,0],[1,0],[0,1],[0,1]]) + assert_array_equal(S_result.todense(),S_expected) + + + class TestFitCandidates(NumpyTestCase): def setUp(self): self.cases = [] Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-15 22:46:49 UTC (rev 3439) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-16 06:19:12 UTC (rev 3440) @@ -3,10 +3,11 @@ import numpy import scipy -from numpy import ravel,arange +from numpy import ravel,arange,concatenate from scipy.linalg import norm from scipy.sparse import isspmatrix,isspmatrix_csr,isspmatrix_csc, \ - csr_matrix,csc_matrix,extract_diagonal + csr_matrix,csc_matrix,extract_diagonal, \ + coo_matrix def approximate_spectral_radius(A,tol=0.1,maxiter=20): From scipy-svn at scipy.org Tue Oct 16 19:40:40 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 16 Oct 2007 18:40:40 -0500 (CDT) Subject: [Scipy-svn] r3441 - trunk/scipy/io Message-ID: <20071016234040.619E039C24D@new.scipy.org> Author: chris.burns Date: 2007-10-16 18:40:37 -0500 (Tue, 16 Oct 2007) New Revision: 3441 Removed: trunk/scipy/io/path.py Modified: trunk/scipy/io/datasource.py Log: Cleanup datasource. Remove non-standard path module. Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-16 06:19:12 UTC (rev 3440) +++ trunk/scipy/io/datasource.py 2007-10-16 23:40:37 UTC (rev 3441) @@ -4,8 +4,8 @@ """ # TODO: Make DataSource and Repository the public interface. -# Cache will be used internally. Add methods in DataSource to expose -# some of the functionality that exists only in Cache currently. +# _Cache will be used internally. Add methods in DataSource to expose +# some of the functionality that exists only in _Cache currently. __docformat__ = "restructuredtext en" @@ -13,12 +13,9 @@ import gzip import bz2 from urlparse import urlparse -from urllib2 import urlopen +from urllib2 import urlopen, URLError from tempfile import mkstemp -# TODO: replace with newer tuple-based path module -from scipy.io.path import path - import warnings # datasource has been used for a while in the NIPY project for analyzing @@ -34,129 +31,88 @@ warnings.warn(_api_warning) -# TODO: .zip support -_zipexts = (".gz",".bz2") +# TODO: Don't check file extensions, just try and open zip files? +# Follow the, it's easier to get forgiveness than permission? +# And because this shouldn't succeed: +# In [134]: datasource._iszip('/my/fake/dir/foobar.gz') +# Out[134]: True + +# TODO: .zip support, .tar support? _file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file} -def iszip(filename): +def _iszip(filename): """Test if the given file is a zip file. - *Parameters*: - - filename : {string} - Filename to test. - - *Returns*: - - bool - Results of test. - + Currently only looks at the file extension, not very robust. """ - _tmp, ext = path(filename).splitext() - return ext in _zipexts + fname, ext = os.path.splitext(filename) + return ext in _file_openers.keys() -def unzip(filename): - """Unzip the given file and return the path object to the new file. +def _unzip(filename): + """Unzip the given file and return the path object to the new file.""" - *Parameters*: - - filename : string - Filename to unzip. - - *Returns*: - - path_obj - Path object of the unzipped file. - - """ - - if not iszip(filename): + # This function is not used in datasource. Appears it was created + # so users could unzip a file without having to import the corresponding + # compression module. Should this be part of datasource? + if not _iszip(filename): raise ValueError("file %s is not zipped"%filename) - unzip_name, zipext = splitzipext(filename) + unzip_name, zipext = _splitzipext(filename) opener = _file_openers[zipext] outfile = file(unzip_name, 'w') outfile.write(opener(filename).read()) outfile.close() return unzip_name -def iswritemode(mode): - """Test if the given mode will open a file for writing. +def _iswritemode(mode): + """Test if the given mode will open a file for writing.""" - *Parameters*: - - mode : {string} - The mode to be checked. - - *Returns*: - - bool - Result of test. - - """ - _writemodes = ("w", "+", "a") for c in mode: if c in _writemodes: return True return False -def splitzipext(filename): +def _splitzipext(filename): """Split the filename into a path object and a zip extension. If the filename does not have a zip extension the zip_ext in the return will be None. - *Parameters*: + Parameters: filename : {string} Filename to split. - *Returns*: + Returns: base, zip_ext : {tuple} Tuple containing a path object to the file and the zip extension. """ - if iszip(filename): - return path(filename).splitext() + if _iszip(filename): + return os.path.splitext(filename) else: return filename, None def _isurl(pathstr): """Test whether a given string can be parsed as a URL. - *Parameters*: - - pathstr : {string} - The string to be checked. + A pathstr with a valid network scheme (http, ftp, ...) will return true. + A pathstr with a 'file' scheme will return false. - *Returns*: - bool - Results of test. - """ - scheme, netloc, _path, _params, _query, _frag = urlparse(pathstr) + scheme, netloc, upath, uparams, uquery, ufrag = urlparse(pathstr) return bool(scheme and netloc) -def ensuredirs(directory): - """Ensure that the given directory path exists. If not, create it. +def _ensuredirs(directory): + """Ensure that the given directory path exists. If not, create it.""" - *Parameters*: - directory : {path object} - - *Returns*: - None + if not os.path.exists(directory): + os.makedirs(directory) - """ - - if not isinstance(directory, path): - directory = path(directory) - if not directory.exists(): - directory.makedirs() - -class Cache (object): +class _Cache (object): """A local file cache for URL datasources. The path of the cache can be specified on intialization. The default @@ -167,29 +123,29 @@ def __init__(self, cachepath=None): if cachepath is not None: - self.path = path(cachepath) + self.path = cachepath elif os.name == 'posix': - self.path = path(os.environ["HOME"]).joinpath(".scipy","cache") + self.path = os.path.join(os.environ["HOME"], ".scipy", "cache") elif os.name == 'nt': - self.path = path(os.environ["HOMEPATH"]).joinpath(".scipy","cache") - if not self.path.exists(): - ensuredirs(self.path) + self.path = os.path.join(os.environ["HOMEPATH"], ".scipy", "cache") + if not os.path.exists(self.path): + _ensuredirs(self.path) def tempfile(self, suffix='', prefix=''): """Create and return a temporary file in the cache. - *Parameters*: + Parameters: suffix : {''}, optional prefix : {''}, optional - *Returns*: + Returns: tmpfile : {string} String containing the full path to the temporary file. - *Examples* + Examples - >>> mycache = datasource.Cache() + >>> mycache = datasource._Cache() >>> mytmpfile = mycache.tempfile() >>> mytmpfile '/home/guido/.scipy/cache/GUPhDv' @@ -202,97 +158,91 @@ def filepath(self, uri): """Return a path object to the uri in the cache. - *Parameters*: + Parameters: uri : {string} Filename to use in the returned path object. - *Returns*: - path_obj - Path object for the given uri. + Returns: + path : {string} + Complete path for the given uri. - *Examples* + Examples - >>> mycache = datasource.Cache() + >>> mycache = datasource._Cache() >>> mycache.filepath('xyzcoords.txt') - path('/home/guido/.scipy/cache/xyzcoords.txt') + '/home/guido/.scipy/cache/xyzcoords.txt' """ - # TODO: Change to non-public? - # It appears the Cache is designed to work with URLs only! - (_tmp, netloc, upath, _tmp, _tmp, _tmp) = urlparse(uri) - return self.path.joinpath(netloc, upath.strip('/')) + scheme, netloc, upath, uparams, uquery, ufrag = urlparse(uri) + return os.path.join(self.path, netloc, upath.strip('/')) - def filename(self, uri): - """Return the complete path + filename within the cache. + def cache(self, uri): + """Copy the file at uri into the cache. - *Parameters*: + Parameters: uri : {string} - Filename to usein the returned path. + path or url of source file to cache. - *Returns*: - filename + Returns: + None - *Examples* - - >>> mycache = datasource.Cache() - >>> mycache.filename('xyzcoords.txt') - '/home/guido/.scipy/cache/xyzcoords.txt' - """ - # TODO: Change to non-public? - - return str(self.filepath(uri)) - def cache(self, uri): - """Copy a file into the cache. - - :Returns: ``None`` - """ if self.iscached(uri): return - print 'cache uri:', uri - upath = self.filepath(uri) + _ensuredirs(os.path.dirname(upath)) - print 'upath:', upath - - ensuredirs(upath.dirname()) - try: - print 'uri:', uri - openedurl = urlopen(uri) - except: - raise IOError("url not found: "+str(uri)) - file(upath, 'w').write(openedurl.read()) + print 'cache - source:', uri + print 'cache - destination:', upath + if _isurl(uri): + try: + openedurl = urlopen(uri) + file(upath, 'w').write(openedurl.read()) + except URLError: + raise URLError("URL not found: " + str(uri)) + else: + try: + fp = file(uri, 'r') + file(upath, 'w').write(fp.read()) + except IOError: + raise IOError("File not founcd: " + str(uri)) + def clear(self): """Delete all files in the cache.""" - + # TODO: This deletes all files in the cache directory, regardless # of if this instance created them. Too destructive and - # unexpected behavior. - - for _file in self.path.files(): - os.remove(file) + # unexpected behavior. + #for _file in self.path.files(): + # os.remove(file) + raise NotImplementedError def iscached(self, uri): """ Check if a file exists in the cache. - :Returns: ``bool`` + Returns + boolean + """ - return self.filepath(uri).exists() + upath = self.filepath(uri) + return os.path.exists(upath) + def retrieve(self, uri): - """ - Retrieve a file from the cache. - If not already there, create the file and - add it to the cache. + """Retrieve a file from the cache. + If not already there, create the file and add it to the cache. - :Returns: ``file`` + Returns + open file object + """ + self.cache(uri) - return file(self.filename(uri)) + return file(self.filepath(uri)) class DataSource (object): @@ -306,21 +256,21 @@ """ def __init__(self, cachepath=os.curdir): - self._cache = Cache(cachepath) + self._cache = _Cache(cachepath) def tempfile(self, suffix='', prefix=''): - """Create a temporary file in the cache. + """Create a temporary file in the DataSource cache. - *Parameters*: + Parameters: suffix : {''}, optional prefix : {''}, optional - *Returns*: + Returns: tmpfile : {string} String containing the full path to the temporary file. - *Examples* + Examples >>> datasrc = datasource.DataSource() >>> tmpfile = datasrc.tempfile() @@ -333,36 +283,55 @@ def _possible_names(self, filename): """Return a tuple containing compressed filenames.""" names = [filename] - if not iszip(filename): - for zipext in _zipexts: + if not _iszip(filename): + for zipext in _file_openers.keys(): names.append(filename+zipext) return tuple(names) def cache(self, pathstr): - # TODO: Should work with files also, not just urls. - if _isurl(pathstr): - self._cache.cache(pathstr) + """Cache the file specified by pathstr. + + Creates a copy of file pathstr in the datasource cache. + + """ + + self._cache.cache(pathstr) def clear(self): # TODO: Implement a clear interface for deleting tempfiles. - # There's a problem with the way this is handled in the Cache, + # There's a problem with the way this is handled in the _Cache, # All files in the cache directory will be deleted. In the # default instance, that's the os.curdir. I doubt this is what # people would want. The instance should only delete files that # it created! - pass + raise NotImplementedError def filename(self, pathstr): + """Searches for pathstr file and returns full path if found. + + If pathstr is an URL, filename will cache a local copy and return + the path to the cached file. + If pathstr is a local file, filename will return a path to that local + file. + BUG: This should be modified so the behavior is identical for both + types of files! + + The search will include possible compressed versions of the files. + BUG: Will return the first occurence found, regardless of which + version is newer. + + """ + found = None for name in self._possible_names(pathstr): try: if _isurl(name): self.cache(name) - found = self._cache.filename(name) + found = self._cache.filepath(name) else: raise Exception except: - if path(name).exists(): + if os.path.exists(name): found = name if found: break @@ -371,25 +340,39 @@ return found def exists(self, pathstr): + """Test if pathstr exists in the cache or the current directory. + + If pathstr is an URL, it will be fetched and cached. + + """ + + # Is this method doing to much? At very least may want an option to + # not fetch and cache URLs. + try: - _ = self.filename(pathstr) + _datafile = self.filename(pathstr) return True except IOError: return False def open(self, pathstr, mode='r'): - if _isurl(pathstr) and iswritemode(mode): + """Open pathstr and return file object. + + If pathstr is an URL, it will be fetched and cached. + + """ + + # Is this method doing to much? Should it be fetching and caching? + + if _isurl(pathstr) and _iswritemode(mode): raise ValueError("URLs are not writeable") found = self.filename(pathstr) - _, ext = splitzipext(found) + _fname, ext = _splitzipext(found) if ext == 'bz2': mode.replace("+", "") return _file_openers[ext](found, mode=mode) - def _fullpath(self, pathstr): - return pathstr - class Repository (DataSource): """Multiple DataSource's that share one base url. @@ -397,20 +380,19 @@ """ - #"""DataSource with an implied root.""" - def __init__(self, baseurl, cachepath=None): DataSource.__init__(self, cachepath=cachepath) self._baseurl = baseurl def _fullpath(self, pathstr): - return path(self._baseurl).joinpath(pathstr) + return os.path.join(self._baseurl, pathstr) def filename(self, pathstr): - return DataSource.filename(self, str(self._fullpath(pathstr))) + return DataSource.filename(self, self._fullpath(pathstr)) def exists(self, pathstr): return DataSource.exists(self, self._fullpath(pathstr)) def open(self, pathstr, mode='r'): return DataSource.open(self, self._fullpath(pathstr), mode) + Deleted: trunk/scipy/io/path.py =================================================================== --- trunk/scipy/io/path.py 2007-10-16 06:19:12 UTC (rev 3440) +++ trunk/scipy/io/path.py 2007-10-16 23:40:37 UTC (rev 3441) @@ -1,966 +0,0 @@ -""" path.py - An object representing a path to a file or directory. - -Example: - -from scipy.io.path import path -d = path('/home/guido/bin') -for f in d.files('*.py'): - f.chmod(0755) - - -URL: http://www.jorendorff.com/articles/python/path -Author: Jason Orendorff (and others - see the url!) -Date: 9 Mar 2007 -""" - - -# TODO -# - Tree-walking functions don't avoid symlink loops. Matt Harrison -# sent me a patch for this. -# - Bug in write_text(). It doesn't support Universal newline mode. -# - Better error message in listdir() when self isn't a -# directory. (On Windows, the error message really sucks.) -# - Make sure everything has a good docstring. -# - Add methods for regex find and replace. -# - guess_content_type() method? -# - Perhaps support arguments to touch(). - -import sys, warnings, os, fnmatch, glob, shutil, codecs, md5 - -__version__ = '2.2' -__all__ = ['path'] - -# Platform-specific support for path.owner -if os.name == 'nt': - try: - import win32security - except ImportError: - win32security = None -else: - try: - import pwd - except ImportError: - pwd = None - -# Pre-2.3 support. Are unicode filenames supported? -_base = str -_getcwd = os.getcwd -try: - if os.path.supports_unicode_filenames: - _base = unicode - _getcwd = os.getcwdu -except AttributeError: - pass - -# Pre-2.3 workaround for booleans -try: - True, False -except NameError: - True, False = 1, 0 - -# Pre-2.3 workaround for basestring. -try: - basestring -except NameError: - basestring = (str, unicode) - -# Universal newline support -_textmode = 'r' -if hasattr(file, 'newlines'): - _textmode = 'U' - - -class TreeWalkWarning(Warning): - pass - -class path(_base): - """ Represents a filesystem path. - - For documentation on individual methods, consult their - counterparts in os.path. - """ - - # --- Special Python methods. - - def __repr__(self): - return 'path(%s)' % _base.__repr__(self) - - # Adding a path and a string yields a path. - def __add__(self, more): - try: - resultStr = _base.__add__(self, more) - except TypeError: #Python bug - resultStr = NotImplemented - if resultStr is NotImplemented: - return resultStr - return self.__class__(resultStr) - - def __radd__(self, other): - if isinstance(other, basestring): - return self.__class__(other.__add__(self)) - else: - return NotImplemented - - # The / operator joins paths. - def __div__(self, rel): - """ fp.__div__(rel) == fp / rel == fp.joinpath(rel) - - Join two path components, adding a separator character if - needed. - """ - return self.__class__(os.path.join(self, rel)) - - # Make the / operator work even when true division is enabled. - __truediv__ = __div__ - - def getcwd(cls): - """ Return the current working directory as a path object. """ - return cls(_getcwd()) - getcwd = classmethod(getcwd) - - - # --- Operations on path strings. - - isabs = os.path.isabs - def abspath(self): return self.__class__(os.path.abspath(self)) - def normcase(self): return self.__class__(os.path.normcase(self)) - def normpath(self): return self.__class__(os.path.normpath(self)) - def realpath(self): return self.__class__(os.path.realpath(self)) - def expanduser(self): return self.__class__(os.path.expanduser(self)) - def expandvars(self): return self.__class__(os.path.expandvars(self)) - def dirname(self): return self.__class__(os.path.dirname(self)) - basename = os.path.basename - - def expand(self): - """ Clean up a filename by calling expandvars(), - expanduser(), and normpath() on it. - - This is commonly everything needed to clean up a filename - read from a configuration file, for example. - """ - return self.expandvars().expanduser().normpath() - - def _get_namebase(self): - base, ext = os.path.splitext(self.name) - return base - - def _get_ext(self): - f, ext = os.path.splitext(_base(self)) - return ext - - def _get_drive(self): - drive, r = os.path.splitdrive(self) - return self.__class__(drive) - - parent = property( - dirname, None, None, - """ This path's parent directory, as a new path object. - - For example, path('/usr/local/lib/libpython.so').parent == path('/usr/local/lib') - """) - - name = property( - basename, None, None, - """ The name of this file or directory without the full path. - - For example, path('/usr/local/lib/libpython.so').name == 'libpython.so' - """) - - namebase = property( - _get_namebase, None, None, - """ The same as path.name, but with one file extension stripped off. - - For example, path('/home/guido/python.tar.gz').name == 'python.tar.gz', - but path('/home/guido/python.tar.gz').namebase == 'python.tar' - """) - - ext = property( - _get_ext, None, None, - """ The file extension, for example '.py'. """) - - drive = property( - _get_drive, None, None, - """ The drive specifier, for example 'C:'. - This is always empty on systems that don't use drive specifiers. - """) - - def splitpath(self): - """ p.splitpath() -> Return (p.parent, p.name). """ - parent, child = os.path.split(self) - return self.__class__(parent), child - - def splitdrive(self): - """ p.splitdrive() -> Return (p.drive, ). - - Split the drive specifier from this path. If there is - no drive specifier, p.drive is empty, so the return value - is simply (path(''), p). This is always the case on Unix. - """ - drive, rel = os.path.splitdrive(self) - return self.__class__(drive), rel - - def splitext(self): - """ p.splitext() -> Return (p.stripext(), p.ext). - - Split the filename extension from this path and return - the two parts. Either part may be empty. - - The extension is everything from '.' to the end of the - last path segment. This has the property that if - (a, b) == p.splitext(), then a + b == p. - """ - filename, ext = os.path.splitext(self) - return self.__class__(filename), ext - - def stripext(self): - """ p.stripext() -> Remove one file extension from the path. - - For example, path('/home/guido/python.tar.gz').stripext() - returns path('/home/guido/python.tar'). - """ - return self.splitext()[0] - - if hasattr(os.path, 'splitunc'): - def splitunc(self): - unc, rest = os.path.splitunc(self) - return self.__class__(unc), rest - - def _get_uncshare(self): - unc, r = os.path.splitunc(self) - return self.__class__(unc) - - uncshare = property( - _get_uncshare, None, None, - """ The UNC mount point for this path. - This is empty for paths on local drives. """) - - def joinpath(self, *args): - """ Join two or more path components, adding a separator - character (os.sep) if needed. Returns a new path - object. - """ - return self.__class__(os.path.join(self, *args)) - - def splitall(self): - r""" Return a list of the path components in this path. - - The first item in the list will be a path. Its value will be - either os.curdir, os.pardir, empty, or the root directory of - this path (for example, '/' or 'C:\\'). The other items in - the list will be strings. - - path.path.joinpath(*result) will yield the original path. - """ - parts = [] - loc = self - while loc != os.curdir and loc != os.pardir: - prev = loc - loc, child = prev.splitpath() - if loc == prev: - break - parts.append(child) - parts.append(loc) - parts.reverse() - return parts - - def relpath(self): - """ Return this path as a relative path, - based from the current working directory. - """ - cwd = self.__class__(os.getcwd()) - return cwd.relpathto(self) - - def relpathto(self, dest): - """ Return a relative path from self to dest. - - If there is no relative path from self to dest, for example if - they reside on different drives in Windows, then this returns - dest.abspath(). - """ - origin = self.abspath() - dest = self.__class__(dest).abspath() - - orig_list = origin.normcase().splitall() - # Don't normcase dest! We want to preserve the case. - dest_list = dest.splitall() - - if orig_list[0] != os.path.normcase(dest_list[0]): - # Can't get here from there. - return dest - - # Find the location where the two paths start to differ. - i = 0 - for start_seg, dest_seg in zip(orig_list, dest_list): - if start_seg != os.path.normcase(dest_seg): - break - i += 1 - - # Now i is the point where the two paths diverge. - # Need a certain number of "os.pardir"s to work up - # from the origin to the point of divergence. - segments = [os.pardir] * (len(orig_list) - i) - # Need to add the diverging part of dest_list. - segments += dest_list[i:] - if len(segments) == 0: - # If they happen to be identical, use os.curdir. - relpath = os.curdir - else: - relpath = os.path.join(*segments) - return self.__class__(relpath) - - # --- Listing, searching, walking, and matching - - def listdir(self, pattern=None): - """ D.listdir() -> List of items in this directory. - - Use D.files() or D.dirs() instead if you want a listing - of just files or just subdirectories. - - The elements of the list are path objects. - - With the optional 'pattern' argument, this only lists - items whose names match the given pattern. - """ - names = os.listdir(self) - if pattern is not None: - names = fnmatch.filter(names, pattern) - return [self / child for child in names] - - def dirs(self, pattern=None): - """ D.dirs() -> List of this directory's subdirectories. - - The elements of the list are path objects. - This does not walk recursively into subdirectories - (but see path.walkdirs). - - With the optional 'pattern' argument, this only lists - directories whose names match the given pattern. For - example, d.dirs('build-*'). - """ - return [p for p in self.listdir(pattern) if p.isdir()] - - def files(self, pattern=None): - """ D.files() -> List of the files in this directory. - - The elements of the list are path objects. - This does not walk into subdirectories (see path.walkfiles). - - With the optional 'pattern' argument, this only lists files - whose names match the given pattern. For example, - d.files('*.pyc'). - """ - - return [p for p in self.listdir(pattern) if p.isfile()] - - def walk(self, pattern=None, errors='strict'): - """ D.walk() -> iterator over files and subdirs, recursively. - - The iterator yields path objects naming each child item of - this directory and its descendants. This requires that - D.isdir(). - - This performs a depth-first traversal of the directory tree. - Each directory is returned just before all its children. - - The errors= keyword argument controls behavior when an - error occurs. The default is 'strict', which causes an - exception. The other allowed values are 'warn', which - reports the error via warnings.warn(), and 'ignore'. - """ - if errors not in ('strict', 'warn', 'ignore'): - raise ValueError("invalid errors parameter") - - try: - childList = self.listdir() - except Exception: - if errors == 'ignore': - return - elif errors == 'warn': - warnings.warn( - "Unable to list directory '%s': %s" - % (self, sys.exc_info()[1]), - TreeWalkWarning) - return - else: - raise - - for child in childList: - if pattern is None or child.fnmatch(pattern): - yield child - try: - isdir = child.isdir() - except Exception: - if errors == 'ignore': - isdir = False - elif errors == 'warn': - warnings.warn( - "Unable to access '%s': %s" - % (child, sys.exc_info()[1]), - TreeWalkWarning) - isdir = False - else: - raise - - if isdir: - for item in child.walk(pattern, errors): - yield item - - def walkdirs(self, pattern=None, errors='strict'): - """ D.walkdirs() -> iterator over subdirs, recursively. - - With the optional 'pattern' argument, this yields only - directories whose names match the given pattern. For - example, mydir.walkdirs('*test') yields only directories - with names ending in 'test'. - - The errors= keyword argument controls behavior when an - error occurs. The default is 'strict', which causes an - exception. The other allowed values are 'warn', which - reports the error via warnings.warn(), and 'ignore'. - """ - if errors not in ('strict', 'warn', 'ignore'): - raise ValueError("invalid errors parameter") - - try: - dirs = self.dirs() - except Exception: - if errors == 'ignore': - return - elif errors == 'warn': - warnings.warn( - "Unable to list directory '%s': %s" - % (self, sys.exc_info()[1]), - TreeWalkWarning) - return - else: - raise - - for child in dirs: - if pattern is None or child.fnmatch(pattern): - yield child - for subsubdir in child.walkdirs(pattern, errors): - yield subsubdir - - def walkfiles(self, pattern=None, errors='strict'): - """ D.walkfiles() -> iterator over files in D, recursively. - - The optional argument, pattern, limits the results to files - with names that match the pattern. For example, - mydir.walkfiles('*.tmp') yields only files with the .tmp - extension. - """ - if errors not in ('strict', 'warn', 'ignore'): - raise ValueError("invalid errors parameter") - - try: - childList = self.listdir() - except Exception: - if errors == 'ignore': - return - elif errors == 'warn': - warnings.warn( - "Unable to list directory '%s': %s" - % (self, sys.exc_info()[1]), - TreeWalkWarning) - return - else: - raise - - for child in childList: - try: - isfile = child.isfile() - isdir = not isfile and child.isdir() - except: - if errors == 'ignore': - continue - elif errors == 'warn': - warnings.warn( - "Unable to access '%s': %s" - % (self, sys.exc_info()[1]), - TreeWalkWarning) - continue - else: - raise - - if isfile: - if pattern is None or child.fnmatch(pattern): - yield child - elif isdir: - for f in child.walkfiles(pattern, errors): - yield f - - def fnmatch(self, pattern): - """ Return True if self.name matches the given pattern. - - pattern - A filename pattern with wildcards, - for example '*.py'. - """ - return fnmatch.fnmatch(self.name, pattern) - - def glob(self, pattern): - """ Return a list of path objects that match the pattern. - - pattern - a path relative to this directory, with wildcards. - - For example, path('/users').glob('*/bin/*') returns a list - of all the files users have in their bin directories. - """ - cls = self.__class__ - return [cls(s) for s in glob.glob(_base(self / pattern))] - - - # --- Reading or writing an entire file at once. - - def open(self, mode='r'): - """ Open this file. Return a file object. """ - return file(self, mode) - - def bytes(self): - """ Open this file, read all bytes, return them as a string. """ - f = self.open('rb') - try: - return f.read() - finally: - f.close() - - def write_bytes(self, bytes, append=False): - """ Open this file and write the given bytes to it. - - Default behavior is to overwrite any existing file. - Call p.write_bytes(bytes, append=True) to append instead. - """ - if append: - mode = 'ab' - else: - mode = 'wb' - f = self.open(mode) - try: - f.write(bytes) - finally: - f.close() - - def text(self, encoding=None, errors='strict'): - r""" Open this file, read it in, return the content as a string. - - This uses 'U' mode in Python 2.3 and later, so '\r\n' and '\r' - are automatically translated to '\n'. - - Optional arguments: - - encoding - The Unicode encoding (or character set) of - the file. If present, the content of the file is - decoded and returned as a unicode object; otherwise - it is returned as an 8-bit str. - errors - How to handle Unicode errors; see help(str.decode) - for the options. Default is 'strict'. - """ - if encoding is None: - # 8-bit - f = self.open(_textmode) - try: - return f.read() - finally: - f.close() - else: - # Unicode - f = codecs.open(self, 'r', encoding, errors) - # (Note - Can't use 'U' mode here, since codecs.open - # doesn't support 'U' mode, even in Python 2.3.) - try: - t = f.read() - finally: - f.close() - return (t.replace(u'\r\n', u'\n') - .replace(u'\r\x85', u'\n') - .replace(u'\r', u'\n') - .replace(u'\x85', u'\n') - .replace(u'\u2028', u'\n')) - - def write_text(self, text, encoding=None, errors='strict', linesep=os.linesep, append=False): - r""" Write the given text to this file. - - The default behavior is to overwrite any existing file; - to append instead, use the 'append=True' keyword argument. - - There are two differences between path.write_text() and - path.write_bytes(): newline handling and Unicode handling. - See below. - - Parameters: - - - text - str/unicode - The text to be written. - - - encoding - str - The Unicode encoding that will be used. - This is ignored if 'text' isn't a Unicode string. - - - errors - str - How to handle Unicode encoding errors. - Default is 'strict'. See help(unicode.encode) for the - options. This is ignored if 'text' isn't a Unicode - string. - - - linesep - keyword argument - str/unicode - The sequence of - characters to be used to mark end-of-line. The default is - os.linesep. You can also specify None; this means to - leave all newlines as they are in 'text'. - - - append - keyword argument - bool - Specifies what to do if - the file already exists (True: append to the end of it; - False: overwrite it.) The default is False. - - - --- Newline handling. - - write_text() converts all standard end-of-line sequences - ('\n', '\r', and '\r\n') to your platform's default end-of-line - sequence (see os.linesep; on Windows, for example, the - end-of-line marker is '\r\n'). - - If you don't like your platform's default, you can override it - using the 'linesep=' keyword argument. If you specifically want - write_text() to preserve the newlines as-is, use 'linesep=None'. - - This applies to Unicode text the same as to 8-bit text, except - there are three additional standard Unicode end-of-line sequences: - u'\x85', u'\r\x85', and u'\u2028'. - - (This is slightly different from when you open a file for - writing with fopen(filename, "w") in C or file(filename, 'w') - in Python.) - - - --- Unicode - - If 'text' isn't Unicode, then apart from newline handling, the - bytes are written verbatim to the file. The 'encoding' and - 'errors' arguments are not used and must be omitted. - - If 'text' is Unicode, it is first converted to bytes using the - specified 'encoding' (or the default encoding if 'encoding' - isn't specified). The 'errors' argument applies only to this - conversion. - - """ - if isinstance(text, unicode): - if linesep is not None: - # Convert all standard end-of-line sequences to - # ordinary newline characters. - text = (text.replace(u'\r\n', u'\n') - .replace(u'\r\x85', u'\n') - .replace(u'\r', u'\n') - .replace(u'\x85', u'\n') - .replace(u'\u2028', u'\n')) - text = text.replace(u'\n', linesep) - if encoding is None: - encoding = sys.getdefaultencoding() - bytes = text.encode(encoding, errors) - else: - # It is an error to specify an encoding if 'text' is - # an 8-bit string. - assert encoding is None - - if linesep is not None: - text = (text.replace('\r\n', '\n') - .replace('\r', '\n')) - bytes = text.replace('\n', linesep) - - self.write_bytes(bytes, append) - - def lines(self, encoding=None, errors='strict', retain=True): - r""" Open this file, read all lines, return them in a list. - - Optional arguments: - encoding - The Unicode encoding (or character set) of - the file. The default is None, meaning the content - of the file is read as 8-bit characters and returned - as a list of (non-Unicode) str objects. - errors - How to handle Unicode errors; see help(str.decode) - for the options. Default is 'strict' - retain - If true, retain newline characters; but all newline - character combinations ('\r', '\n', '\r\n') are - translated to '\n'. If false, newline characters are - stripped off. Default is True. - - This uses 'U' mode in Python 2.3 and later. - """ - if encoding is None and retain: - f = self.open(_textmode) - try: - return f.readlines() - finally: - f.close() - else: - return self.text(encoding, errors).splitlines(retain) - - def write_lines(self, lines, encoding=None, errors='strict', - linesep=os.linesep, append=False): - r""" Write the given lines of text to this file. - - By default this overwrites any existing file at this path. - - This puts a platform-specific newline sequence on every line. - See 'linesep' below. - - lines - A list of strings. - - encoding - A Unicode encoding to use. This applies only if - 'lines' contains any Unicode strings. - - errors - How to handle errors in Unicode encoding. This - also applies only to Unicode strings. - - linesep - The desired line-ending. This line-ending is - applied to every line. If a line already has any - standard line ending ('\r', '\n', '\r\n', u'\x85', - u'\r\x85', u'\u2028'), that will be stripped off and - this will be used instead. The default is os.linesep, - which is platform-dependent ('\r\n' on Windows, '\n' on - Unix, etc.) Specify None to write the lines as-is, - like file.writelines(). - - Use the keyword argument append=True to append lines to the - file. The default is to overwrite the file. Warning: - When you use this with Unicode data, if the encoding of the - existing data in the file is different from the encoding - you specify with the encoding= parameter, the result is - mixed-encoding data, which can really confuse someone trying - to read the file later. - """ - if append: - mode = 'ab' - else: - mode = 'wb' - f = self.open(mode) - try: - for line in lines: - isUnicode = isinstance(line, unicode) - if linesep is not None: - # Strip off any existing line-end and add the - # specified linesep string. - if isUnicode: - if line[-2:] in (u'\r\n', u'\x0d\x85'): - line = line[:-2] - elif line[-1:] in (u'\r', u'\n', - u'\x85', u'\u2028'): - line = line[:-1] - else: - if line[-2:] == '\r\n': - line = line[:-2] - elif line[-1:] in ('\r', '\n'): - line = line[:-1] - line += linesep - if isUnicode: - if encoding is None: - encoding = sys.getdefaultencoding() - line = line.encode(encoding, errors) - f.write(line) - finally: - f.close() - - def read_md5(self): - """ Calculate the md5 hash for this file. - - This reads through the entire file. - """ - f = self.open('rb') - try: - m = md5.new() - while True: - d = f.read(8192) - if not d: - break - m.update(d) - finally: - f.close() - return m.digest() - - # --- Methods for querying the filesystem. - - exists = os.path.exists - isdir = os.path.isdir - isfile = os.path.isfile - islink = os.path.islink - ismount = os.path.ismount - - if hasattr(os.path, 'samefile'): - samefile = os.path.samefile - - getatime = os.path.getatime - atime = property( - getatime, None, None, - """ Last access time of the file. """) - - getmtime = os.path.getmtime - mtime = property( - getmtime, None, None, - """ Last-modified time of the file. """) - - if hasattr(os.path, 'getctime'): - getctime = os.path.getctime - ctime = property( - getctime, None, None, - """ Creation time of the file. """) - - getsize = os.path.getsize - size = property( - getsize, None, None, - """ Size of the file, in bytes. """) - - if hasattr(os, 'access'): - def access(self, mode): - """ Return true if current user has access to this path. - - mode - One of the constants os.F_OK, os.R_OK, os.W_OK, os.X_OK - """ - return os.access(self, mode) - - def stat(self): - """ Perform a stat() system call on this path. """ - return os.stat(self) - - def lstat(self): - """ Like path.stat(), but do not follow symbolic links. """ - return os.lstat(self) - - def get_owner(self): - r""" Return the name of the owner of this file or directory. - - This follows symbolic links. - - On Windows, this returns a name of the form ur'DOMAIN\User Name'. - On Windows, a group can own a file or directory. - """ - if os.name == 'nt': - if win32security is None: - raise Exception("path.owner requires win32all to be installed") - desc = win32security.GetFileSecurity( - self, win32security.OWNER_SECURITY_INFORMATION) - sid = desc.GetSecurityDescriptorOwner() - account, domain, typecode = win32security.LookupAccountSid(None, sid) - return domain + u'\\' + account - else: - if pwd is None: - raise NotImplementedError("path.owner is not implemented on this platform.") - st = self.stat() - return pwd.getpwuid(st.st_uid).pw_name - - owner = property( - get_owner, None, None, - """ Name of the owner of this file or directory. """) - - if hasattr(os, 'statvfs'): - def statvfs(self): - """ Perform a statvfs() system call on this path. """ - return os.statvfs(self) - - if hasattr(os, 'pathconf'): - def pathconf(self, name): - return os.pathconf(self, name) - - - # --- Modifying operations on files and directories - - def utime(self, times): - """ Set the access and modified times of this file. """ - os.utime(self, times) - - def chmod(self, mode): - os.chmod(self, mode) - - if hasattr(os, 'chown'): - def chown(self, uid, gid): - os.chown(self, uid, gid) - - def rename(self, new): - os.rename(self, new) - - def renames(self, new): - os.renames(self, new) - - - # --- Create/delete operations on directories - - def mkdir(self, mode=0777): - os.mkdir(self, mode) - - def makedirs(self, mode=0777): - os.makedirs(self, mode) - - def rmdir(self): - os.rmdir(self) - - def removedirs(self): - os.removedirs(self) - - - # --- Modifying operations on files - - def touch(self): - """ Set the access/modified times of this file to the current time. - Create the file if it does not exist. - """ - fd = os.open(self, os.O_WRONLY | os.O_CREAT, 0666) - os.close(fd) - os.utime(self, None) - - def remove(self): - os.remove(self) - - def unlink(self): - os.unlink(self) - - - # --- Links - - if hasattr(os, 'link'): - def link(self, newpath): - """ Create a hard link at 'newpath', pointing to this file. """ - os.link(self, newpath) - - if hasattr(os, 'symlink'): - def symlink(self, newlink): - """ Create a symbolic link at 'newlink', pointing here. """ - os.symlink(self, newlink) - - if hasattr(os, 'readlink'): - def readlink(self): - """ Return the path to which this symbolic link points. - - The result may be an absolute or a relative path. - """ - return self.__class__(os.readlink(self)) - - def readlinkabs(self): - """ Return the path to which this symbolic link points. - - The result is always an absolute path. - """ - p = self.readlink() - if p.isabs(): - return p - else: - return (self.parent / p).abspath() - - - # --- High-level functions from shutil - - copyfile = shutil.copyfile - copymode = shutil.copymode - copystat = shutil.copystat - copy = shutil.copy - copy2 = shutil.copy2 - copytree = shutil.copytree - if hasattr(shutil, 'move'): - move = shutil.move - rmtree = shutil.rmtree - - - # --- Special stuff from os - - if hasattr(os, 'chroot'): - def chroot(self): - os.chroot(self) - - if hasattr(os, 'startfile'): - def startfile(self): - os.startfile(self) - From scipy-svn at scipy.org Wed Oct 17 04:13:14 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 17 Oct 2007 03:13:14 -0500 (CDT) Subject: [Scipy-svn] r3442 - trunk Message-ID: <20071017081314.C311739C2A5@new.scipy.org> Author: jarrod.millman Date: 2007-10-17 03:13:12 -0500 (Wed, 17 Oct 2007) New Revision: 3442 Modified: trunk/setup.py Log: added info for PKG-INFO Modified: trunk/setup.py =================================================================== --- trunk/setup.py 2007-10-16 23:40:37 UTC (rev 3441) +++ trunk/setup.py 2007-10-17 08:13:12 UTC (rev 3442) @@ -1,7 +1,42 @@ #!/usr/bin/env python +"""SciPy: Scientific Library for Python + +SciPy (pronounced "Sigh Pie") is open-source software for mathematics, +science, and engineering. The SciPy library +depends on NumPy, which provides convenient and fast N-dimensional +array manipulation. The SciPy library is built to work with NumPy +arrays, and provides many user-friendly and efficient numerical +routines such as routines for numerical integration and optimization. +Together, they run on all popular operating systems, are quick to +install, and are free of charge. NumPy and SciPy are easy to use, +but powerful enough to be depended upon by some of the world's +leading scientists and engineers. If you need to manipulate +numbers on a computer and display or publish the results, +give SciPy a try! + +""" + +DOCLINES = __doc__.split("\n") + import os import sys +CLASSIFIERS = """\ +Development Status :: 4 - Beta +Intended Audience :: Science/Research +Intended Audience :: Developers +License :: OSI Approved +Programming Language :: C +Programming Language :: Python +Topic :: Software Development +Topic :: Scientific/Engineering +Operating System :: Microsoft :: Windows +Operating System :: POSIX +Operating System :: Unix +Operating System :: MacOS + +""" + # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly # update it when the contents of directories change. if os.path.exists('MANIFEST'): os.remove('MANIFEST') @@ -39,9 +74,13 @@ name = 'scipy', maintainer = "SciPy Developers", maintainer_email = "scipy-dev at scipy.org", - description = "Scientific Algorithms Library for Python", + description = DOCLINES[0], + long_description = "\n".join(DOCLINES[2:]), url = "http://www.scipy.org", + download_url = "http://sourceforge.net/project/showfiles.php?group_id=27747&package_id=19531", license = 'BSD', + classifiers=filter(None, CLASSIFIERS.split('\n')), + platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], configuration=configuration ) finally: del sys.path[0] From scipy-svn at scipy.org Wed Oct 17 14:28:35 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 17 Oct 2007 13:28:35 -0500 (CDT) Subject: [Scipy-svn] r3443 - in trunk/scipy/stats/models: robust tests Message-ID: <20071017182835.3A30239C018@new.scipy.org> Author: jonathan.taylor Date: 2007-10-17 13:28:32 -0500 (Wed, 17 Oct 2007) New Revision: 3443 Added: trunk/scipy/stats/models/tests/test_scale.py Modified: trunk/scipy/stats/models/robust/scale.py Log: added axis option to MAD estimate, and Huber estimate of scale added test for scipy.models.robust.scale Modified: trunk/scipy/stats/models/robust/scale.py =================================================================== --- trunk/scipy/stats/models/robust/scale.py 2007-10-17 08:13:12 UTC (rev 3442) +++ trunk/scipy/stats/models/robust/scale.py 2007-10-17 18:28:32 UTC (rev 3443) @@ -1,19 +1,41 @@ import numpy as N -from scipy import median -from scipy.stats import norm +from scipy.stats import norm, median -def MAD(a, c=0.6745): +def unsqueeze(data, axis, oldshape): """ - Median Absolute Deviation along first axis of an array: + unsqueeze a collapsed array + >>> from numpy import mean + >>> from numpy.random import standard_normal + >>> x = standard_normal((3,4,5)) + >>> m = mean(x, axis=1) + >>> m.shape + (3, 5) + >>> unsqueeze(m, 1, x.shape) + >>> m.shape + (3, 1, 5) + >>> + """ + + newshape = list(oldshape) + newshape[axis] = 1 + data.shape = newshape + + +def MAD(a, c=0.6745, axis=0): + """ + Median Absolute Deviation along given axis of an array: + median(abs(a - median(a))) / c """ a = N.asarray(a, N.float64) - d = N.multiply.outer(median(a), N.ones(a.shape[1:])) - return median(N.fabs(a - d) / c) + d = median(a, axis=axis) + unsqueeze(d, axis, a.shape) + return median(N.fabs(a - d) / c, axis=axis) + class Huber: """ Huber's proposal 2 for estimating scale. @@ -29,19 +51,20 @@ gamma = tmp + c**2 * (1 - tmp) - 2 * c * norm.pdf(c) del(tmp) - niter = 10 + niter = 30 - def __call__(self, a, mu=None, scale=None): + def __call__(self, a, mu=None, scale=None, axis=0): """ Compute Huber\'s proposal 2 estimate of scale, using an optional initial value of scale and an optional estimate of mu. If mu is supplied, it is not reestimated. """ + self.axis = axis self.a = N.asarray(a, N.float64) if mu is None: self.n = self.a.shape[0] - 1 - self.mu = N.multiply.outer(median(self.a), N.ones(self.a.shape[1:])) + self.mu = median(self.a, axis=axis) self.est_mu = True else: self.n = self.a.shape[0] @@ -49,14 +72,18 @@ self.est_mu = False if scale is None: - self.scale = MAD(self.a)**2 + self.scale = MAD(self.a, axis=self.axis)**2 else: self.scale = scale + unsqueeze(self.scale, self.axis, self.a.shape) + unsqueeze(self.mu, self.axis, self.a.shape) + for donothing in self: pass - self.s = N.sqrt(self.scale) + self.s = N.squeeze(N.sqrt(self.scale)) + del(self.scale); del(self.mu); del(self.a) return self.s def __iter__(self): @@ -67,16 +94,16 @@ a = self.a subset = self.subset(a) if self.est_mu: - mu = (subset * a).sum() / a.shape[0] + mu = N.sum(subset * a + (1 - Huber.c) * subset, axis=self.axis) / a.shape[self.axis] else: mu = self.mu + unsqueeze(mu, self.axis, self.a.shape) - scale = N.sum(subset * (a - mu)**2, axis=0) / (self.n * Huber.gamma - N.sum(1. - subset, axis=0) * Huber.c**2) + scale = N.sum(subset * (a - mu)**2, axis=self.axis) / (self.n * Huber.gamma - N.sum(1. - subset, axis=self.axis) * Huber.c**2) self.iter += 1 - if (N.fabs(N.sqrt(scale) - N.sqrt(self.scale)) <= N.sqrt(self.scale) * Huber.tol and - N.fabs(mu - self.mu) <= N.sqrt(self.scale) * Huber.tol): + if N.alltrue(N.less_equal(N.fabs(N.sqrt(scale) - N.sqrt(self.scale)), N.sqrt(self.scale) * Huber.tol)) and N.alltrue(N.less_equal(N.fabs(mu - self.mu), N.sqrt(self.scale) * Huber.tol)): self.scale = scale self.mu = mu raise StopIteration @@ -84,6 +111,8 @@ self.scale = scale self.mu = mu + unsqueeze(self.scale, self.axis, self.a.shape) + if self.iter >= self.niter: raise StopIteration Added: trunk/scipy/stats/models/tests/test_scale.py =================================================================== --- trunk/scipy/stats/models/tests/test_scale.py 2007-10-17 08:13:12 UTC (rev 3442) +++ trunk/scipy/stats/models/tests/test_scale.py 2007-10-17 18:28:32 UTC (rev 3443) @@ -0,0 +1,53 @@ +""" +Test functions for models.robust.scale +""" + +import numpy.random as R +from numpy.testing import NumpyTest, NumpyTestCase + +import scipy.stats.models.robust.scale as scale + +W = R.standard_normal + +class TestScale(NumpyTestCase): + + def test_MAD(self): + X = W((40,10)) + m = scale.MAD(X) + self.assertEquals(m.shape, (10,)) + + def test_MADaxes(self): + X = W((40,10,30)) + m = scale.MAD(X, axis=0) + self.assertEquals(m.shape, (10,30)) + + m = scale.MAD(X, axis=1) + self.assertEquals(m.shape, (40,30)) + + m = scale.MAD(X, axis=2) + self.assertEquals(m.shape, (40,10)) + + m = scale.MAD(X, axis=-1) + self.assertEquals(m.shape, (40,10)) + + def test_huber(self): + X = W((40,10)) + m = scale.huber(X) + self.assertEquals(m.shape, (10,)) + + def test_huberaxes(self): + X = W((40,10,30)) + m = scale.huber(X, axis=0) + self.assertEquals(m.shape, (10,30)) + + m = scale.huber(X, axis=1) + self.assertEquals(m.shape, (40,30)) + + m = scale.huber(X, axis=2) + self.assertEquals(m.shape, (40,10)) + + m = scale.huber(X, axis=-1) + self.assertEquals(m.shape, (40,10)) + +if __name__ == "__main__": + NumpyTest().run() From scipy-svn at scipy.org Wed Oct 17 16:29:17 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 17 Oct 2007 15:29:17 -0500 (CDT) Subject: [Scipy-svn] r3444 - in trunk/scipy/sandbox/multigrid: . tests Message-ID: <20071017202917.B796D39C0B9@new.scipy.org> Author: wnbell Date: 2007-10-17 15:29:14 -0500 (Wed, 17 Oct 2007) New Revision: 3444 Modified: trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/tests/test_sa.py Log: added tests cases for user-defined aggregation Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-17 18:28:32 UTC (rev 3443) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-17 20:29:14 UTC (rev 3444) @@ -77,9 +77,8 @@ candidates = [ ones(A.shape[0]) ] # use constant vector if aggregation is None: - while len(As) < max_levels and A.shape[0] > max_coarse: - P,candidates = sa_interpolation(A,candidates,epsilon*0.5**(len(As)-1),omega=omega,blocks=blocks) - blocks = None #only used for 1st level + while len(As) < max_levels and A.shape[0] > max_coarse: + P,candidates,blocks = sa_interpolation(A,candidates,epsilon*0.5**(len(As)-1),omega=omega,blocks=blocks) A = (P.T.tocsr() * A) * P #galerkin operator @@ -207,12 +206,12 @@ candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] blocks = arange(A.shape[0]/2).repeat(2) - ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,max_coarse=10,max_levels=10) + ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,epsilon=0,max_coarse=10,max_levels=10) #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) - b = zeros_like(x) - #b = rand(A.shape[0]) + #b = zeros_like(x) + b = A*rand(A.shape[0]) if True: x_sol,residuals = ml.solve(b,x0=x,maxiter=30,tol=1e-12,return_residuals=True) @@ -221,7 +220,7 @@ def add_resid(x): residuals.append(linalg.norm(b - A*x)) A.psolve = ml.psolve - x_sol = linalg.cg(A,b,x0=x,maxiter=12,tol=1e-100,callback=add_resid)[0] + x_sol = linalg.cg(A,b,x0=x,maxiter=25,tol=1e-12,callback=add_resid)[0] residuals = array(residuals)/residuals[0] @@ -233,3 +232,5 @@ + + Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-17 18:28:32 UTC (rev 3443) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-17 20:29:14 UTC (rev 3444) @@ -8,7 +8,7 @@ import multigridtools __all__ = ['sa_filtered_matrix','sa_strong_connections','sa_constant_interpolation', - 'sa_interpolation','sa_fit_candidates'] + 'sa_interpolation','sa_smoothed_prolongator','sa_fit_candidates'] def sa_filtered_matrix(A,epsilon,blocks=None): @@ -83,6 +83,7 @@ Pj = Pj[blocks] #expand block aggregates into constituent dofs Pp = B.indptr Px = B.data + else: S = sa_strong_connections(A,epsilon) @@ -140,7 +141,18 @@ return Q,coarse_candidates - +def sa_smoothed_prolongator(A,T,epsilon,omega,blocks=None): + A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems + + D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) + D_inv_A = D_inv * A_filtered + D_inv_A *= omega/approximate_spectral_radius(D_inv_A) + + # smooth tentative prolongator T + P = T - (D_inv_A*T) + + return P + def sa_interpolation(A,candidates,epsilon=0.0,omega=4.0/3.0,blocks=None,AggOp=None): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') @@ -148,7 +160,7 @@ AggOp = sa_constant_interpolation(A,epsilon=epsilon,blocks=blocks) else: if not isspmatrix_csr(AggOp): - raise TypeError,'aggregation is specified by a list of csr_matrix objects' + raise TypeError,'expected csr_matrix for argument AggOp' if A.shape[1] != AggOp.shape[0]: raise ValueError,'incompatible aggregation operator' @@ -156,14 +168,19 @@ A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems - D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) - D_inv_A = D_inv * A_filtered - D_inv_A *= omega/approximate_spectral_radius(D_inv_A) + P = sa_smoothed_prolongator(A,T,epsilon,omega,blocks) - # smooth tentative prolongator T - P = T - (D_inv_A*T) - - return P,coarse_candidates +## D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) +## D_inv_A = D_inv * A_filtered +## D_inv_A *= omega/approximate_spectral_radius(D_inv_A) +## +## # smooth tentative prolongator T +## P = T - (D_inv_A*T) + if blocks is not None: + blocks = arange(AggOp.shape[1]).repeat(len(candidates)) + + return P,coarse_candidates,blocks + Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-17 18:28:32 UTC (rev 3443) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-17 20:29:14 UTC (rev 3444) @@ -12,7 +12,8 @@ set_package_path() import scipy.sandbox.multigrid from scipy.sandbox.multigrid.sa import sa_strong_connections, sa_constant_interpolation, \ - sa_interpolation, sa_fit_candidates + sa_interpolation, sa_fit_candidates, \ + sa_smoothed_prolongator from scipy.sandbox.multigrid.multilevel import poisson_problem1D,poisson_problem2D, \ smoothed_aggregation_solver from scipy.sandbox.multigrid.utils import diag_sparse @@ -68,6 +69,15 @@ S_result = sa_constant_interpolation(A,epsilon,blocks=arange(A.shape[0])) assert_array_equal(S_result.todense(),S_expected.todense()) + # two aggregates in 1D + A = poisson_problem1D(6) + AggOp = csr_matrix((ones(6),array([0,0,0,1,1,1]),arange(7)),dims=(6,2)) + candidates = [ones(6)] + + T_result,coarse_candidates_result = sa_fit_candidates(AggOp,candidates) + T_expected = csr_matrix((sqrt(1.0/3.0)*ones(6),array([0,0,0,1,1,1]),arange(7)),dims=(6,2)) + assert_almost_equal(T_result.todense(),T_expected.todense()) + #check simple block examples A = csr_matrix(arange(16).reshape(4,4)) A = A + A.T @@ -85,6 +95,32 @@ S_expected = matrix([[1,0],[1,0],[0,1],[0,1]]) assert_array_equal(S_result.todense(),S_expected) + + def check_user_aggregation(self): + """check that the sa_interpolation accepts user-defined aggregates""" + + user_cases = [] + + #simple 1d example w/ two aggregates + A = poisson_problem1D(6) + AggOp = csr_matrix((ones(6),array([0,0,0,1,1,1]),arange(7)),dims=(6,2)) + candidates = [ones(6)] + user_cases.append((A,AggOp,candidates)) + + #simple 1d example w/ two aggregates (not all nodes are aggregated) + A = poisson_problem1D(6) + AggOp = csr_matrix((ones(4),array([0,0,1,1]),array([0,1,1,2,3,3,4])),dims=(6,2)) + candidates = [ones(6)] + user_cases.append((A,AggOp,candidates)) + + for A,AggOp,candidates in user_cases: + T,coarse_candidates_result = sa_fit_candidates(AggOp,candidates) + + P_result = sa_interpolation(A,candidates,omega=4.0/3.0,AggOp=AggOp)[0] + P_expected = sa_smoothed_prolongator(A,T,epsilon=0.0,omega=4.0/3.0) + + assert_almost_equal(P_result.todense(),P_expected.todense()) + class TestFitCandidates(NumpyTestCase): @@ -149,9 +185,7 @@ - - -class TestSASolver(NumpyTestCase): +class TestSASolverPerformance(NumpyTestCase): def setUp(self): self.cases = [] From scipy-svn at scipy.org Thu Oct 18 04:06:14 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 18 Oct 2007 03:06:14 -0500 (CDT) Subject: [Scipy-svn] r3445 - trunk/scipy/sandbox/multigrid Message-ID: <20071018080614.75E6AC7C052@new.scipy.org> Author: wnbell Date: 2007-10-18 03:06:05 -0500 (Thu, 18 Oct 2007) New Revision: 3445 Added: trunk/scipy/sandbox/multigrid/dec_test.py Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/utils.py Log: fixed bug in expansion of AggOp added dec example, incomplete Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-17 20:29:14 UTC (rev 3444) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-18 08:06:05 UTC (rev 3445) @@ -294,7 +294,7 @@ print "solving" if True: - x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=10,tol=1e-12,return_residuals=True) + x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=25,tol=1e-8,return_residuals=True) else: residuals = [] def add_resid(x): Added: trunk/scipy/sandbox/multigrid/dec_test.py =================================================================== --- trunk/scipy/sandbox/multigrid/dec_test.py 2007-10-17 20:29:14 UTC (rev 3444) +++ trunk/scipy/sandbox/multigrid/dec_test.py 2007-10-18 08:06:05 UTC (rev 3445) @@ -0,0 +1,174 @@ + +from scipy import * +from pydec import * +from pydec.multigrid import * +from pydec.multigrid.discrete_laplacian import boundary_hierarchy, discrete_laplacian_solver, hodge_solver + +from scipy.sandbox.multigrid import smoothed_aggregation_solver +from scipy.sandbox.multigrid.utils import expand_into_blocks + + +## Load mesh from file +mesh_path = '../../../../../hirani_group/wnbell/meshes/' +#mesh = read_mesh(mesh_path + 'rocket/rocket.xml') +#mesh = read_mesh(mesh_path + 'genus3/genus3_168k.xml') +#mesh = read_mesh(mesh_path + 'genus3/genus3_455k.xml') +mesh = read_mesh(mesh_path + '/torus/torus.xml') +for i in range(3): + mesh['vertices'],mesh['elements'] = loop_subdivision(mesh['vertices'],mesh['elements']) +cmplx = simplicial_complex(mesh['vertices'],mesh['elements']) + +## Construct mesh manually +#bitmap = ones((60,60),dtype='bool') +#bitmap[1::10,1::10] = False +#bitmap[100:150,100:400] = False +#cmplx = regular_cube_complex(regular_cube_mesh(bitmap)) + + + +def whitney_innerproduct_cache(cmplx,k): + h = hash(cmplx.vertices.tostring()) ^ hash(cmplx.simplices.tostring()) ^ hash(k) + + filename = "/home/nathan/.pydec/cache/whitney_" + str(h) + ".mtx" + + try: + import pydec + M = pydec.io.read_array(filename) + except: + import pydec + M = whitney_innerproduct(cmplx,k) + pydec.io.write_array(filename,M) + + return M + + + +def cube_innerproduct_cache(cmplx,k): + h = hash(cmplx.mesh.bitmap.tostring()) ^ hash(cmplx.mesh.bitmap.shape) ^ hash(k) + + filename = "/home/nathan/.pydec/cache/cube_" + str(h) + ".mtx" + + try: + import pydec + M = pydec.io.read_array(filename) + except: + import pydec + M = regular_cube_innerproduct(cmplx,k) + pydec.io.write_array(filename,M) + + return M + + + +#solve d_k d_k problem for all reasonable k +#from pylab import semilogy,show,xlabel,ylabel,legend,ylim,xlim +#from matplotlib.font_manager import fontManager, FontProperties + +cochain_complex = cmplx.cochain_complex() + +for i in [1]: #range(len(cochain_complex)-1): + print "computing mass matrix" + + if isinstance(cmplx,simplicial_complex): + Mi = whitney_innerproduct_cache(cmplx,i+1) + else: + Mi = regular_cube_innerproduct(cmplx,i+1) + + ##print "constructing solver" + ##ss = discrete_laplacian_solver(cochain_complex,len(cochain_complex)-i-1,innerproduct=Mi) + ##print ss + ## + ##print "solving" + ##x,res = ss.solve(b=zeros(ss.A.shape[0]),x0=rand(ss.A.shape[0]),return_residuals=True) + + bh = boundary_hierarchy(cochain_complex) + while len(bh) < 3: + bh.coarsen() + print repr(bh) + + N = len(cochain_complex) - 1 + + B = bh[0][N - i].B + + A = B.T.tocsr() * B + #A = B.T.tocsr() * Mi * B + + constant_prolongators = [lvl[N - i].I for lvl in bh[:-1]] + + if i == 0: + candidates = None + else: + #candidates = [ones(A.shape[0])] + + #TODO test + candidates = [] + for coord in range(mesh['vertices'].shape[1]): + candidates.append( bh[0][N-i+1].B * mesh['vertices'][:,coord] ) + + K = len(candidates) + + constant_prolongators = [constant_prolongators[0]] + \ + [expand_into_blocks(T,K,1).tocsr() for T in constant_prolongators[1:] ] + + + ml = smoothed_aggregation_solver(A,candidates,aggregation=constant_prolongators) + #ml = smoothed_aggregation_solver(A,candidates) + + x = rand(A.shape[0]) + b = zeros_like(x) + #b = A*rand(A.shape[0]) + + if True: + x_sol,residuals = ml.solve(b,x0=x,maxiter=50,tol=1e-12,return_residuals=True) + else: + residuals = [] + def add_resid(x): + residuals.append(linalg.norm(b - A*x)) + A.psolve = ml.psolve + x_sol = linalg.cg(A,b,x0=x,maxiter=30,tol=1e-12,callback=add_resid)[0] + + + residuals = array(residuals)/residuals[0] + avg_convergence_ratio = residuals[-1]**(1.0/len(residuals)) + print "average convergence ratio",avg_convergence_ratio + print "last convergence ratio",residuals[-1]/residuals[-2] + + print residuals + + + + +##candidates = None +##blocks = None +## +## +## +##A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() +##candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') +##candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] +##blocks = arange(A.shape[0]/2).repeat(2) +## +##ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,epsilon=0,max_coarse=10,max_levels=10) +###ml = ruge_stuben_solver(A) +## +##x = rand(A.shape[0]) +###b = zeros_like(x) +##b = A*rand(A.shape[0]) +## +##if True: +## x_sol,residuals = ml.solve(b,x0=x,maxiter=30,tol=1e-12,return_residuals=True) +##else: +## residuals = [] +## def add_resid(x): +## residuals.append(linalg.norm(b - A*x)) +## A.psolve = ml.psolve +## x_sol = linalg.cg(A,b,x0=x,maxiter=25,tol=1e-12,callback=add_resid)[0] +## +## +##residuals = array(residuals)/residuals[0] +##avg_convergence_ratio = residuals[-1]**(1.0/len(residuals)) +##print "average convergence ratio",avg_convergence_ratio +##print "last convergence ratio",residuals[-1]/residuals[-2] +## +##print residuals +## Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-17 20:29:14 UTC (rev 3444) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-18 08:06:05 UTC (rev 3445) @@ -87,7 +87,7 @@ else: #use user-defined aggregation for AggOp in aggregation: - P,candidates = sa_interpolation(A,candidates,omega=omega,AggOp=AggOp) + P,candidates,blocks = sa_interpolation(A,candidates,omega=omega,AggOp=AggOp) A = (P.T.tocsr() * A) * P #galerkin operator @@ -210,11 +210,11 @@ #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) - #b = zeros_like(x) - b = A*rand(A.shape[0]) + b = zeros_like(x) + #b = A*rand(A.shape[0]) if True: - x_sol,residuals = ml.solve(b,x0=x,maxiter=30,tol=1e-12,return_residuals=True) + x_sol,residuals = ml.solve(b,x0=x,maxiter=30,tol=1e-8,return_residuals=True) else: residuals = [] def add_resid(x): Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-17 20:29:14 UTC (rev 3444) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-18 08:06:05 UTC (rev 3445) @@ -2,28 +2,53 @@ import numpy from numpy import array,arange,ones,zeros,sqrt,isinf,asarray,empty,diff,\ ascontiguousarray -from scipy.sparse import csr_matrix,isspmatrix_csr +from scipy.sparse import csr_matrix,isspmatrix_csr,spidentity -from utils import diag_sparse,approximate_spectral_radius +from utils import diag_sparse,approximate_spectral_radius,expand_into_blocks import multigridtools __all__ = ['sa_filtered_matrix','sa_strong_connections','sa_constant_interpolation', 'sa_interpolation','sa_smoothed_prolongator','sa_fit_candidates'] +## nnz = A.nnz +## +## indptr = A.indptr +## indices = A.indices +## data = A.data +## +## if n != 1: +## # expand horizontally +## indptr = n*A.indptr +## indices = (n*indices).repeat(n) + tile(arange(n),nnz) +## +## if m != 1: +## #expand vertically +## indptr = concatenate( (array([0]), cumsum(diff(A).repeat(m))) ) +## indices = indices.repeat(m) +## +## if m != 1 or n != 1: +## data = A.data.repeat(m*n) + + def sa_filtered_matrix(A,epsilon,blocks=None): + """The filtered matrix is obtained from A by lumping all weak off-diagonal + entries onto the diagonal. Weak off-diagonals are determined by + the standard strength of connection measure using the parameter epsilon. + + In the case epsilon = 0.0, (i.e. no weak connections) A is returned. + """ + if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') if epsilon == 0: A_filtered = A - else: if blocks is None: Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) A_filtered = csr_matrix((Sx,Sj,Sp),A.shape) else: A_filtered = A #TODO subtract weak blocks from diagonal blocks? - ## num_dofs = A.shape[0] ## num_blocks = blocks.max() + 1 ## @@ -50,7 +75,6 @@ return A_filtered - def sa_strong_connections(A,epsilon): if not isspmatrix_csr(A): raise TypeError('expected csr_matrix') @@ -101,7 +125,9 @@ if K > 1 and len(candidates[0]) == K*N_fine: #see if fine space has been expanded (all levels except for first) - AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) + #TODO fix this and add unittests + #AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) + AggOp = expand_into_blocks(AggOp,K,1).tocsr() N_fine = K*N_fine #TODO convert this to list of coarse candidates @@ -142,6 +168,20 @@ return Q,coarse_candidates def sa_smoothed_prolongator(A,T,epsilon,omega,blocks=None): + """For a given matrix A and tentative prolongator T return the + smoothed prolongator P + + P = (I - omega/rho(S) S) * T + + where S is a Jacobi smoothing operator defined as follows: + + omega - damping parameter + rho(S) - spectral radius of S (estimated) + S - inv(diag(A_filtered)) * A_filtered (Jacobi smoother) + A_filtered - sa_filtered_matrix(A,epsilon) + """ + + A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) @@ -150,6 +190,9 @@ # smooth tentative prolongator T P = T - (D_inv_A*T) + + #S = (spidentity(A.shape[0]).tocsr() - D_inv_A) #TODO drop this? + #P = S * ( S * T) return P @@ -163,20 +206,15 @@ raise TypeError,'expected csr_matrix for argument AggOp' if A.shape[1] != AggOp.shape[0]: raise ValueError,'incompatible aggregation operator' + T,coarse_candidates = sa_fit_candidates(AggOp,candidates) + #T = AggOp #TODO test A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems P = sa_smoothed_prolongator(A,T,epsilon,omega,blocks) -## D_inv = diag_sparse(1.0/diag_sparse(A_filtered)) -## D_inv_A = D_inv * A_filtered -## D_inv_A *= omega/approximate_spectral_radius(D_inv_A) -## -## # smooth tentative prolongator T -## P = T - (D_inv_A*T) - if blocks is not None: blocks = arange(AggOp.shape[1]).repeat(len(candidates)) Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-17 20:29:14 UTC (rev 3444) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-18 08:06:05 UTC (rev 3445) @@ -1,9 +1,9 @@ __all__ =['approximate_spectral_radius','infinity_norm','diag_sparse', - 'hstack_csr','vstack_csr'] + 'hstack_csr','vstack_csr','expand_into_blocks'] import numpy import scipy -from numpy import ravel,arange,concatenate +from numpy import ravel,arange,concatenate,tile from scipy.linalg import norm from scipy.sparse import isspmatrix,isspmatrix_csr,isspmatrix_csc, \ csr_matrix,csc_matrix,extract_diagonal, \ @@ -67,3 +67,47 @@ V = concatenate((A.data,B.data)) return coo_matrix((V,(I,J)),dims=(A.shape[0]+B.shape[0],A.shape[1])).tocsr() + +def expand_into_blocks(A,m,n): + """Expand each element in a sparse matrix A into an m-by-n block. + + Example: + >>> A.todense() + matrix([[ 1., 2.], + [ 4., 5.]]) + + >>> expand_into_blocks(A,2,2).todense() + matrix([[ 1., 1., 2., 2.], + [ 1., 1., 2., 2.], + [ 4., 4., 5., 5.], + [ 4., 4., 5., 5.]]) + + """ + #TODO EXPLAIN MORE + + if n is None: + n = m + + if m == 1 and n == 1: + return A #nothing to do + + A = A.tocoo() + + # expand 1x1 -> mxn + row = ( m*A.row ).repeat(m*n).reshape(-1,m,n) + col = ( n*A.col ).repeat(m*n).reshape(-1,m,n) + + # increment indices + row += tile(arange(m).reshape(-1,1),(1,n)) + col += tile(arange(n).reshape(1,-1),(m,1)) + + # flatten + row = row.reshape(-1) + col = col.reshape(-1) + + data = A.data.repeat(m*n) + + return coo_matrix((data,(row,col)),dims=(m*A.shape[0],n*A.shape[1])) + + + From scipy-svn at scipy.org Thu Oct 18 18:25:44 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 18 Oct 2007 17:25:44 -0500 (CDT) Subject: [Scipy-svn] r3446 - trunk/scipy/io/tests Message-ID: <20071018222544.6345939C049@new.scipy.org> Author: matthew.brett at gmail.com Date: 2007-10-18 17:25:30 -0500 (Thu, 18 Oct 2007) New Revision: 3446 Modified: trunk/scipy/io/tests/test_mio.py Log: Fixed tests to use only 3d matrix matlab 5 round trip Modified: trunk/scipy/io/tests/test_mio.py =================================================================== --- trunk/scipy/io/tests/test_mio.py 2007-10-18 08:06:05 UTC (rev 3445) +++ trunk/scipy/io/tests/test_mio.py 2007-10-18 22:25:30 UTC (rev 3446) @@ -89,7 +89,7 @@ def _make_rt_check_case(name, expected, format): def cc(self): mat_stream = StringIO() - savemat(mat_stream, expected, format) + savemat(mat_stream, expected, format=format) mat_stream.seek(0) self._check_case(name, [mat_stream], expected) cc.__doc__ = "check loadmat case %s" % name @@ -162,6 +162,10 @@ {'name': '3dmatrix', 'expected': {'test3dmatrix': transpose(reshape(range(1,25), (4,3,2)))} }) + case_table5_rt = [ + {'name': '3dmatrix', + 'expected': {'test3dmatrix': transpose(reshape(range(1,25), (4,3,2)))} + }] st = mat_struct() st.stringfield = u'Rats live on no evil star.' st.doublefield = array([sqrt(2),exp(1),pi]) @@ -225,7 +229,7 @@ assert files, "No files for test %s using filter %s" % (name, filt) exec 'check_%s = _make_check_case(name, files, expected)' % name # round trip tests - for case in case_table4 + case_table5: + for case in case_table4 + case_table5_rt: name = case['name'] + '_round_trip' expected = case['expected'] format = case in case_table4 and '4' or '5' From scipy-svn at scipy.org Fri Oct 19 14:05:31 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 19 Oct 2007 13:05:31 -0500 (CDT) Subject: [Scipy-svn] r3447 - trunk/scipy/sparse Message-ID: <20071019180531.A6F69C7C030@new.scipy.org> Author: rkern Date: 2007-10-19 13:05:30 -0500 (Fri, 19 Oct 2007) New Revision: 3447 Modified: trunk/scipy/sparse/sparse.py Log: Remove generator expression for 2.3 compatibility. Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-18 22:25:30 UTC (rev 3446) +++ trunk/scipy/sparse/sparse.py 2007-10-19 18:05:30 UTC (rev 3447) @@ -175,8 +175,8 @@ def listprint(self, start, stop): """Provides a way to print over a single index. """ - return '\n'.join(' %s\t%s' % (self.rowcol(ind), self.getdata(ind)) - for ind in xrange(start,stop)) + '\n' + return '\n'.join([' %s\t%s' % (self.rowcol(ind), self.getdata(ind)) + for ind in xrange(start,stop)]) + '\n' def __repr__(self): nnz = self.getnnz() From scipy-svn at scipy.org Fri Oct 19 17:54:30 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 19 Oct 2007 16:54:30 -0500 (CDT) Subject: [Scipy-svn] r3448 - in trunk/scipy/sparse: . tests Message-ID: <20071019215430.3606E39C0E8@new.scipy.org> Author: wnbell Date: 2007-10-19 16:54:27 -0500 (Fri, 19 Oct 2007) New Revision: 3448 Modified: trunk/scipy/sparse/sparse.py trunk/scipy/sparse/tests/test_sparse.py Log: added sparse.spkron(a,b) - sparse kronecker product resolves ticket #42 Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-19 18:05:30 UTC (rev 3447) +++ trunk/scipy/sparse/sparse.py 2007-10-19 21:54:27 UTC (rev 3448) @@ -7,7 +7,7 @@ __all__ = ['spmatrix','csc_matrix','csr_matrix','coo_matrix', 'lil_matrix','dok_matrix', - 'spdiags','speye','spidentity','extract_diagonal', + 'spdiags','speye','spidentity','spkron','extract_diagonal', 'isspmatrix','issparse','isspmatrix_csc','isspmatrix_csr', 'isspmatrix_lil','isspmatrix_dok', 'isspmatrix_coo', 'lil_eye', 'lil_diags' ] @@ -2854,6 +2854,65 @@ diags = ones((1, n), dtype = dtype) return spdiags(diags, k, n, m) +def spkron(a,b): + """kronecker product of sparse matrices a and b + in COOrdinate format. + + *Parameters*: + a,b : sparse matrices + E.g. csr_matrix, csc_matrix, coo_matrix, etc. + + *Returns*: + coo_matrix + kronecker product in COOrdinate format + + *Example*: + ------- + + >>> a = csr_matrix(array([[0,2],[5,0]])) + >>> b = csr_matrix(array([[1,2],[3,4]])) + >>> spkron(a,b).todense() + matrix([[ 0., 0., 2., 4.], + [ 0., 0., 6., 8.], + [ 5., 10., 0., 0.], + [ 15., 20., 0., 0.]]) + + """ + if not isspmatrix(a) and isspmatrix(b): + raise ValueError,'expected sparse matrix' + + a,b = a.tocoo(),b.tocoo() + output_shape = (a.shape[0]*b.shape[0],a.shape[1]*b.shape[1]) + + if a.nnz == 0 or b.nnz == 0: + # kronecker product is the zero matrix + return coo_matrix(None, dims=output_shape) + + + # expand entries of a into blocks + row = a.row.repeat(b.nnz) + col = a.col.repeat(b.nnz) + data = a.data.repeat(b.nnz) + + row *= b.shape[0] + col *= b.shape[1] + + # increment block indices + row,col = row.reshape(-1,b.nnz),col.reshape(-1,b.nnz) + row += b.row + col += b.col + row,col = row.reshape(-1),col.reshape(-1) + + # compute block entries + data = data.reshape(-1,b.nnz) + data *= b.data + data = data.reshape(-1) + + return coo_matrix((data,(row,col)), dims=output_shape) + + + + def lil_eye((r,c), k=0, dtype=float): """Generate a lil_matrix of dimensions (r,c) with the k-th diagonal set to 1. Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-19 18:05:30 UTC (rev 3447) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-19 21:54:27 UTC (rev 3448) @@ -22,7 +22,7 @@ from numpy.testing import * set_package_path() from scipy.sparse import csc_matrix, csr_matrix, dok_matrix, coo_matrix, \ - spidentity, speye, extract_diagonal, lil_matrix, lil_eye, lil_diags + spidentity, speye, spkron, extract_diagonal, lil_matrix, lil_eye, lil_diags from scipy.linsolve import splu restore_path() @@ -1072,9 +1072,32 @@ b = array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype='d') assert_array_equal(a.toarray(), b) - + def check_spkron(self): + from numpy import kron -class TestCoo(NumpyTestCase): + cases = [] + + cases.append(array([[ 0]])) + cases.append(array([[-1]])) + cases.append(array([[ 4]])) + cases.append(array([[10]])) + cases.append(array([[0],[0]])) + cases.append(array([[0,0]])) + cases.append(array([[1,2],[3,4]])) + cases.append(array([[0,2],[5,0]])) + cases.append(array([[0,2,-6],[8,0,14]])) + cases.append(array([[5,4],[0,0],[6,0]])) + cases.append(array([[5,4,4],[1,0,0],[6,0,8]])) + cases.append(array([[0,1,0,2,0,5,8]])) + + for a in cases: + for b in cases: + result = spkron(csr_matrix(a),csr_matrix(b)).todense() + expected = kron(a,b) + + assert_array_equal(result,expected) + +class TestCOO(NumpyTestCase): def check_constructor1(self): row = numpy.array([2, 3, 1, 3, 0, 1, 3, 0, 2, 1, 2]) col = numpy.array([0, 1, 0, 0, 1, 1, 2, 2, 2, 2, 1]) From scipy-svn at scipy.org Fri Oct 19 23:16:34 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 19 Oct 2007 22:16:34 -0500 (CDT) Subject: [Scipy-svn] r3449 - trunk/scipy/sparse Message-ID: <20071020031634.A93DA39C0A7@new.scipy.org> Author: wnbell Date: 2007-10-19 22:16:30 -0500 (Fri, 19 Oct 2007) New Revision: 3449 Modified: trunk/scipy/sparse/sparse.py Log: small docstring edit Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-19 21:54:27 UTC (rev 3448) +++ trunk/scipy/sparse/sparse.py 2007-10-20 03:16:30 UTC (rev 3449) @@ -2855,8 +2855,7 @@ return spdiags(diags, k, n, m) def spkron(a,b): - """kronecker product of sparse matrices a and b - in COOrdinate format. + """kronecker product of sparse matrices a and b *Parameters*: a,b : sparse matrices From scipy-svn at scipy.org Sun Oct 21 16:56:04 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 21 Oct 2007 15:56:04 -0500 (CDT) Subject: [Scipy-svn] r3450 - trunk/scipy/ndimage/src Message-ID: <20071021205604.578FB39C030@new.scipy.org> Author: matthew.brett at gmail.com Date: 2007-10-21 15:55:46 -0500 (Sun, 21 Oct 2007) New Revision: 3450 Modified: trunk/scipy/ndimage/src/nd_image.h Log: Removed obscure function pointer defines causing crashes with gcc 4.2 in the hope this will reveal their purpose Modified: trunk/scipy/ndimage/src/nd_image.h =================================================================== --- trunk/scipy/ndimage/src/nd_image.h 2007-10-20 03:16:30 UTC (rev 3449) +++ trunk/scipy/ndimage/src/nd_image.h 2007-10-21 20:55:46 UTC (rev 3450) @@ -273,12 +273,6 @@ NA_ByteOrder(), 1, 1); } -#define NA_OutputArray (*(PyArrayObject* (*) (PyObject*,NumarrayType,int) ) (void *) NA_OutputArray) -#define NA_IoArray (*(PyArrayObject* (*) (PyObject*,NumarrayType,int) ) (void *) NA_IoArray) -#define NA_NewArray (*(PyArrayObject* (*) (void* buffer, NumarrayType, int, ...) ) (void *) NA_NewArray ) -#define NA_elements (*(unsigned long (*) (PyArrayObject*) ) (void *) NA_elements) -#define NA_InputArray (*(PyArrayObject* (*) (PyObject*,NumarrayType,int) ) (void *) NA_InputArray) - #endif /* ND_IMPORT_ARRAY */ #endif /* ND_IMAGE_H */ From scipy-svn at scipy.org Sun Oct 21 20:21:18 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 21 Oct 2007 19:21:18 -0500 (CDT) Subject: [Scipy-svn] r3451 - in trunk/scipy/sparse: . tests Message-ID: <20071022002118.EAAB139C0B6@new.scipy.org> Author: wnbell Date: 2007-10-21 19:21:14 -0500 (Sun, 21 Oct 2007) New Revision: 3451 Modified: trunk/scipy/sparse/sparse.py trunk/scipy/sparse/tests/test_sparse.py Log: fix sparse matrix * dense matrix multiplicationa now returns dense matrix Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-21 20:55:46 UTC (rev 3450) +++ trunk/scipy/sparse/sparse.py 2007-10-22 00:21:14 UTC (rev 3451) @@ -17,7 +17,7 @@ from numpy import zeros, isscalar, real, imag, asarray, asmatrix, matrix, \ ndarray, amax, amin, rank, conj, searchsorted, ndarray, \ less, where, greater, array, transpose, empty, ones, \ - arange, shape, intc, clip, prod, unravel_index + arange, shape, intc, clip, prod, unravel_index, hstack import numpy from scipy.sparse.sparsetools import cscmux, csrmux, \ cootocsr, csrtocoo, cootocsc, csctocoo, csctocsr, csrtocsc, \ @@ -345,8 +345,7 @@ def dot(self, other): """ A generic interface for matrix-matrix or matrix-vector - multiplication. Returns A.transpose().conj() * other or - A.transpose() * other. + multiplication. """ try: @@ -355,23 +354,16 @@ # If it's a list or whatever, treat it like a matrix other = asmatrix(other) - if len(other.shape) == 1: - result = self.matvec(other) - elif isdense(other) and asarray(other).squeeze().ndim <= 1: - # If it's a row or column vector, return a DENSE result - result = self.matvec(other) + if isdense(other) and asarray(other).squeeze().ndim <= 1: + # it's a dense row or column vector + return self.matvec(other) elif len(other.shape) == 2: - # Return a sparse result - result = self.matmat(other) + # it's a 2d dense array, dense matrix, or sparse matrix + return self.matmat(other) else: raise ValueError, "could not interpret dimensions" + - if isinstance(other, matrix) and isdense(result): - return asmatrix(result) - else: - # if the result is sparse or 'other' is an array: - return result - def matmat(self, other): csc = self.tocsc() return csc.matmat(other) @@ -663,9 +655,10 @@ other = self._tothis(other) return self._binopt(other,fn,in_shape=(M,N),out_shape=(M,N)) elif isdense(other): - # This is SLOW! We need a more efficient implementation - # of sparse * dense matrix multiplication! - return self.matmat(csc_matrix(other)) + # TODO make sparse * dense matrix multiplication more efficient + + # matvec each column of other + return hstack( [ self * col.reshape(-1,1) for col in other.T ] ) else: raise TypeError, "need a dense or sparse matrix" Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-21 20:55:46 UTC (rev 3450) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-22 00:21:14 UTC (rev 3451) @@ -213,40 +213,40 @@ # Currently M.matvec(asarray(col)) is rank-1, whereas M.matvec(col) # is rank-2. Is this desirable? - def check_matmat(self): + def check_matmat_sparse(self): a = matrix([[3,0,0],[0,1,0],[2,0,3.0],[2,3,0]]) a2 = array([[3,0,0],[0,1,0],[2,0,3.0],[2,3,0]]) b = matrix([[0,1],[1,0],[0,2]],'d') asp = self.spmatrix(a) bsp = self.spmatrix(b) assert_array_almost_equal((asp*bsp).todense(), a*b) - assert_array_almost_equal((asp*b).todense(), a*b) - assert_array_almost_equal((a*bsp).todense(), a*b) - assert_array_almost_equal((a2*bsp).todense(), a*b) + assert_array_almost_equal( asp*b, a*b) + assert_array_almost_equal( a*bsp, a*b) + assert_array_almost_equal( a2*bsp, a*b) # Now try performing cross-type multplication: csp = bsp.tocsc() c = b assert_array_almost_equal((asp*csp).todense(), a*c) assert_array_almost_equal((asp.matmat(csp)).todense(), a*c) - assert_array_almost_equal((asp*c).todense(), a*c) + assert_array_almost_equal( asp*c, a*c) - assert_array_almost_equal((a*csp).todense(), a*c) - assert_array_almost_equal((a2*csp).todense(), a*c) + assert_array_almost_equal( a*csp, a*c) + assert_array_almost_equal( a2*csp, a*c) csp = bsp.tocsr() assert_array_almost_equal((asp*csp).todense(), a*c) assert_array_almost_equal((asp.matmat(csp)).todense(), a*c) - assert_array_almost_equal((asp*c).todense(), a*c) + assert_array_almost_equal( asp*c, a*c) - assert_array_almost_equal((a*csp).todense(), a*c) - assert_array_almost_equal((a2*csp).todense(), a*c) + assert_array_almost_equal( a*csp, a*c) + assert_array_almost_equal( a2*csp, a*c) csp = bsp.tocoo() assert_array_almost_equal((asp*csp).todense(), a*c) assert_array_almost_equal((asp.matmat(csp)).todense(), a*c) - assert_array_almost_equal((asp*c).todense(), a*c) + assert_array_almost_equal( asp*c, a*c) - assert_array_almost_equal((a*csp).todense(), a*c) - assert_array_almost_equal((a2*csp).todense(), a*c) + assert_array_almost_equal( a*csp, a*c) + assert_array_almost_equal( a2*csp, a*c) # Test provided by Andy Fraser, 2006-03-26 L = 30 @@ -262,6 +262,18 @@ assert_array_almost_equal(B.todense(), A.todense() * A.T.todense()) assert_array_almost_equal(B.todense(), A.todense() * A.todense().T) + def check_matmat_dense(self): + a = matrix([[3,0,0],[0,1,0],[2,0,3.0],[2,3,0]]) + asp = self.spmatrix(a) + + # check both array and matrix types + bs = [ array([[1,2],[3,4],[5,6]]), matrix([[1,2],[3,4],[5,6]]) ] + + for b in bs: + result = asp*b + assert( isinstance(result, type(b)) ) + assert_equal( result.shape, (4,2) ) + assert_equal( result, dot(a,b) ) def check_tocoo(self): a = self.datsp.tocoo() From scipy-svn at scipy.org Sun Oct 21 21:18:52 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 21 Oct 2007 20:18:52 -0500 (CDT) Subject: [Scipy-svn] r3452 - in trunk/scipy/sandbox/multigrid: . multigridtools tests Message-ID: <20071022011852.0EAA639C037@new.scipy.org> Author: wnbell Date: 2007-10-21 20:18:47 -0500 (Sun, 21 Oct 2007) New Revision: 3452 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/multigridtools/relaxation.h trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/tests/test_sa.py trunk/scipy/sandbox/multigrid/tests/test_utils.py trunk/scipy/sandbox/multigrid/utils.py Log: made relaxation method ignore rows with zero diagonals changed SA candidates to use dense 2D array Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-22 01:18:47 UTC (rev 3452) @@ -66,13 +66,13 @@ -def sa_hierarchy(A,Ws,x): +def sa_hierarchy(A,Ws,B): """ Construct multilevel hierarchy using Smoothed Aggregation Inputs: A - matrix Is - list of constant prolongators - x - "candidate" basis function to be approximated + B - "candidate" basis function to be approximated Ouputs: (As,Is,Ps) - tuple of lists - As - [A, Ps[0].T*A*Ps[0], Ps[1].T*A*Ps[1], ... ] @@ -84,7 +84,7 @@ Ps = [] for W in Ws: - P,x = sa_fit_candidates(W,x) + P,B = sa_fit_candidates(W,B) I = smoothed_prolongator(P,A) A = I.T.tocsr() * A * I As.append(A) @@ -109,19 +109,21 @@ raise ValueError,'small matrices not handled yet' #first candidate - x,AggOps = self.__initialization_stage(A, blocks=blocks,\ - max_levels=max_levels, max_coarse=max_coarse,\ - mu=mu, epsilon=epsilon) + x,AggOps = self.__initialization_stage(A, blocks = blocks, \ + max_levels = max_levels, \ + max_coarse = max_coarse, \ + mu = mu, epsilon = epsilon) Ws = AggOps self.candidates = [x] #create SA using x here - As,Is,Ps = sa_hierarchy(A,Ws,self.candidates) + As,Is,Ps, = sa_hierarchy(A,AggOps,self.candidates) for i in range(max_candidates - 1): - x = self.__develop_candidate(A,As,Is,Ps,Ws,AggOps,mu=mu) + #x = self.__develop_candidate(A,As,Is,Ps,Ws,AggOps,mu=mu) + x = self.__develop_candidate(As,Is,Ps,AggOps,self.candidates,mu=mu) self.candidates.append(x) @@ -189,9 +191,7 @@ return x,AggOps #first candidate,aggregation - - - def __develop_candidate(self,A,As,Is,Ps,Ws,AggOps,mu): + def __develop_candidate(self,As,Is,Ps,AggOps,candidates,mu): #scipy.random.seed(0) #TEST x = scipy.rand(A.shape[0]) b = zeros_like(x) @@ -201,7 +201,9 @@ x = solver.solve(b, x0=x, tol=1e-10, maxiter=mu) #TEST FOR CONVERGENCE HERE - + + #TODO augment candiates each time, then call fit_candidates? + A_l,P_l,W_l,x_l = As[0],Ps[0],Ws[0],x temp_Is = [] @@ -229,6 +231,46 @@ x = I*x return x + + +## def __develop_candidate(self,A,As,Is,Ps,Ws,AggOps,mu): +## #scipy.random.seed(0) #TEST +## x = scipy.rand(A.shape[0]) +## b = zeros_like(x) +## +## solver = multilevel_solver(As,Is) +## +## x = solver.solve(b, x0=x, tol=1e-10, maxiter=mu) +## +## #TEST FOR CONVERGENCE HERE +## +## A_l,P_l,W_l,x_l = As[0],Ps[0],Ws[0],x +## +## temp_Is = [] +## for i in range(len(As) - 2): +## P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, AggOps[i+1]) +## +## I_l_new = smoothed_prolongator(P_l_new,A_l) +## A_m_new = I_l_new.T.tocsr() * A_l * I_l_new +## bridge = make_bridge(Is[i+1],A_m_new.shape[0]) +## +## temp_solver = multilevel_solver( [A_m_new] + As[i+2:], [bridge] + Is[i+2:] ) +## +## for n in range(mu): +## x_m = temp_solver.solve(zeros_like(x_m), x0=x_m, tol=1e-8, maxiter=1) +## +## temp_Is.append(I_l_new) +## +## W_l = vstack_csr(Ws[i+1],W_m_new) #prepare for next iteration +## A_l = A_m_new +## x_l = x_m +## P_l = make_bridge(Ps[i+1],A_m_new.shape[0]) +## +## x = x_l +## for I in reversed(temp_Is): +## x = I*x +## +## return x def __augment_cycle(self,A,As,Ps,Ws,AggOps,x): @@ -286,15 +328,16 @@ A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() blocks = arange(A.shape[0]/2).repeat(2) -asa = adaptive_sa_solver(A,max_candidates=4,mu=12) +asa = adaptive_sa_solver(A,max_candidates=4,mu=10) scipy.random.seed(0) #make tests repeatable x = rand(A.shape[0]) -b = A*rand(A.shape[0]) +#b = A*rand(A.shape[0]) +b = zeros(A.shape[0]) print "solving" if True: - x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=25,tol=1e-8,return_residuals=True) + x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=25,tol=1e-7,return_residuals=True) else: residuals = [] def add_resid(x): Modified: trunk/scipy/sandbox/multigrid/multigridtools/relaxation.h =================================================================== --- trunk/scipy/sandbox/multigrid/multigridtools/relaxation.h 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/multigridtools/relaxation.h 2007-10-22 01:18:47 UTC (rev 3452) @@ -29,9 +29,10 @@ rsum += Ax[jj]*x[j]; } - assert(diag != 0); - - x[i] = (b[i] - rsum)/diag; + //TODO raise error? inform user? + if (diag != 0){ + x[i] = (b[i] - rsum)/diag; + } } } @@ -64,9 +65,10 @@ rsum += Ax[jj]*temp[j]; } - assert(diag != 0); - - x[i] = (1 - omega) * temp[i] + omega * ((b[i] - rsum)/diag); + //TODO raise error? inform user? + if (diag != 0){ + x[i] = (1 - omega) * temp[i] + omega * ((b[i] - rsum)/diag); + } } } Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-22 01:18:47 UTC (rev 3452) @@ -61,20 +61,53 @@ return multilevel_solver(As,Ps) def smoothed_aggregation_solver(A,candidates=None,blocks=None,aggregation=None,max_levels=10,max_coarse=500,epsilon=0.08,omega=4.0/3.0): - """ - Create a multilevel solver using Smoothed Aggregation (SA) + """Create a multilevel solver using Smoothed Aggregation (SA) - References: - "Algebraic Multigrid by Smoothed Aggregation for Second and Fourth Order Elliptic Problems", - Petr Vanek and Jan Mandel and Marian Brezina - http://citeseer.ist.psu.edu/vanek96algebraic.html + *Parameters*: + + A : {csr_matrix} + NxN matrix in CSR format + B : {None, array_like} : optional + Near-nullspace candidates stored in the columns of an NxK array. + The default value B=None is equivalent to B=ones((N,1)) + blocks : {None, array_like} : optional + Array of length N that groups the variables into 'superblocks'. + For example, in a 2d vector-valued problem where the even + variables [0,2,4,...N-2] correspond to the x-components and the + odd variables [1,3,5,...,N-1] correspond to the y-components then + blocks=[0,0,1,1,2,2,...,N/2,N/2] is expected. The default + value blocks=None is equivalent to blocks=[0,1,2,..,N] which + implies that each variable should be aggregated seperately. + The default is appropriate for scalar valued problems. + aggregation: {None, list of csr_matrix} : optional + List of csr_matrix objects that describe a user-defined + multilevel aggregation of the variables. + TODO ELABORATE + max_levels: {integer} : optional + Maximum number of levels to be used in the multilevel solver. + max_coarse: {integer} : optional + Maximum number of variables permitted on the coarse grid. + epsilon: {float} : optional + Strength of connection parameter used in aggregation. + omega: {float} : optional + Damping parameter used in prolongator smoothing (0 < omega < 2) + + *Example*: + TODO + + *References*: + "Algebraic Multigrid by Smoothed Aggregation for Second and Fourth Order Elliptic Problems", + Petr Vanek and Jan Mandel and Marian Brezina + http://citeseer.ist.psu.edu/vanek96algebraic.html """ As = [A] Ps = [] if candidates is None: - candidates = [ ones(A.shape[0]) ] # use constant vector + candidates = ones((A.shape[0],1),dtype=A.dtype) # use constant vector + else: + candiates = asarray(candidates) if aggregation is None: while len(As) < max_levels and A.shape[0] > max_coarse: @@ -203,10 +236,10 @@ A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') - candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] + #candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] blocks = arange(A.shape[0]/2).repeat(2) - ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,epsilon=0,max_coarse=10,max_levels=10) + ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,epsilon=0,max_coarse=100,max_levels=2) #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-22 01:18:47 UTC (rev 3452) @@ -11,26 +11,7 @@ 'sa_interpolation','sa_smoothed_prolongator','sa_fit_candidates'] -## nnz = A.nnz -## -## indptr = A.indptr -## indices = A.indices -## data = A.data -## -## if n != 1: -## # expand horizontally -## indptr = n*A.indptr -## indices = (n*indices).repeat(n) + tile(arange(n),nnz) -## -## if m != 1: -## #expand vertically -## indptr = concatenate( (array([0]), cumsum(diff(A).repeat(m))) ) -## indices = indices.repeat(m) -## -## if m != 1 or n != 1: -## data = A.data.repeat(m*n) - def sa_filtered_matrix(A,epsilon,blocks=None): """The filtered matrix is obtained from A by lumping all weak off-diagonal entries onto the diagonal. Weak off-diagonals are determined by @@ -48,7 +29,8 @@ Sp,Sj,Sx = multigridtools.sa_strong_connections(A.shape[0],epsilon,A.indptr,A.indices,A.data) A_filtered = csr_matrix((Sx,Sj,Sp),A.shape) else: - A_filtered = A #TODO subtract weak blocks from diagonal blocks? + raise NotImplementedError,'blocks not handled yet' +## #TODO subtract weak blocks from diagonal blocks? ## num_dofs = A.shape[0] ## num_blocks = blocks.max() + 1 ## @@ -99,6 +81,7 @@ B = csr_matrix((ones(num_dofs),blocks,arange(num_dofs + 1)),dims=(num_dofs,num_blocks)) #1-norms of blocks entries of A + #TODO figure out what to do for blocks here Block_A = B.T.tocsr() * csr_matrix((abs(A.data),A.indices,A.indptr),dims=A.shape) * B S = sa_strong_connections(Block_A,epsilon) @@ -119,22 +102,22 @@ def sa_fit_candidates(AggOp,candidates): - K = len(candidates) + + K = candidates.shape[1] # num candidates N_fine,N_coarse = AggOp.shape - if K > 1 and len(candidates[0]) == K*N_fine: + if K > 1 and candidates.shape[0] == K*N_fine: #see if fine space has been expanded (all levels except for first) - #TODO fix this and add unittests - #AggOp = csr_matrix((AggOp.data.repeat(K),AggOp.indices.repeat(K),arange(K*N_fine + 1)),dims=(K*N_fine,N_coarse)) AggOp = expand_into_blocks(AggOp,K,1).tocsr() N_fine = K*N_fine - #TODO convert this to list of coarse candidates R = zeros((K*N_coarse,K)) #storage for coarse candidates candidate_matrices = [] - for i,c in enumerate(candidates): + + for i in range(K): + c = candidates[:,i] c = c[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) @@ -154,6 +137,7 @@ candidate_matrices.append(X) + # expand AggOp blocks horizontally Q_indptr = K*AggOp.indptr Q_indices = (K*AggOp.indices).repeat(K) for i in range(K): @@ -163,9 +147,7 @@ Q_data[i::K] = X.data Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) - coarse_candidates = [ascontiguousarray(R[:,i]) for i in range(K)] - - return Q,coarse_candidates + return Q,R def sa_smoothed_prolongator(A,T,epsilon,omega,blocks=None): """For a given matrix A and tentative prolongator T return the @@ -216,7 +198,7 @@ P = sa_smoothed_prolongator(A,T,epsilon,omega,blocks) if blocks is not None: - blocks = arange(AggOp.shape[1]).repeat(len(candidates)) + blocks = arange(AggOp.shape[1]).repeat(candidates.shape[1]) return P,coarse_candidates,blocks Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-22 01:18:47 UTC (rev 3452) @@ -1,7 +1,7 @@ from numpy.testing import * from numpy import sqrt,empty,ones,arange,array_split,eye,array, \ - zeros,diag,zeros_like,diff,matrix + zeros,diag,zeros_like,diff,matrix,hstack,vstack from numpy.linalg import norm from scipy import rand from scipy.sparse import spdiags,csr_matrix,lil_matrix, \ @@ -72,7 +72,7 @@ # two aggregates in 1D A = poisson_problem1D(6) AggOp = csr_matrix((ones(6),array([0,0,0,1,1,1]),arange(7)),dims=(6,2)) - candidates = [ones(6)] + candidates = ones((6,1)) T_result,coarse_candidates_result = sa_fit_candidates(AggOp,candidates) T_expected = csr_matrix((sqrt(1.0/3.0)*ones(6),array([0,0,0,1,1,1]),arange(7)),dims=(6,2)) @@ -104,13 +104,13 @@ #simple 1d example w/ two aggregates A = poisson_problem1D(6) AggOp = csr_matrix((ones(6),array([0,0,0,1,1,1]),arange(7)),dims=(6,2)) - candidates = [ones(6)] + candidates = ones((6,1)) user_cases.append((A,AggOp,candidates)) #simple 1d example w/ two aggregates (not all nodes are aggregated) A = poisson_problem1D(6) AggOp = csr_matrix((ones(4),array([0,0,1,1]),array([0,1,1,2,3,3,4])),dims=(6,2)) - candidates = [ones(6)] + candidates = ones((6,1)) user_cases.append((A,AggOp,candidates)) for A,AggOp,candidates in user_cases: @@ -129,29 +129,28 @@ ## tests where AggOp includes all DOFs #one candidate - self.cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)),[ones(5)])) - self.cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)),[ones(5)])) - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9)])) - self.cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)),[arange(9)])) + self.cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)), ones((5,1)) )) + self.cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)), ones((5,1)) )) + self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), ones((9,1)) )) + self.cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)), arange(9).reshape(9,1) )) #two candidates - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)),[ones(4),arange(4)])) - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[ones(9),arange(9)])) - self.cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)),[ones(9),arange(9)])) + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)), vstack((ones(4),arange(4))).T )) + self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((ones(9),arange(9))).T )) + self.cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)), vstack((ones(9),arange(9))).T )) #block candidates - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)),[array([1]*9 + [0]*9),arange(2*9)])) + self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((array([1]*9 + [0]*9),arange(2*9))).T )) ## tests where AggOp excludes some DOFs - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)),[ones(5)])) - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)),[ones(5),arange(5)])) - self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)),[ones(9),arange(9)])) + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), ones((5,1)) )) + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5))).T )) + self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9))).T )) def check_all_cases(self): """Test case where aggregation includes all fine nodes""" def mask_candidate(AggOp,candidates): #mask out all DOFs that are not included in the aggregation - for c in candidates: - c[diff(AggOp.indptr) == 0] = 0 + candidates[diff(AggOp.indptr) == 0,:] = 0 for AggOp,fine_candidates in self.cases: @@ -159,32 +158,25 @@ Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) - assert_equal(len(coarse_candidates),len(fine_candidates)) - #each fine level candidate should be fit exactly - for fine,coarse in zip(fine_candidates,coarse_candidates): - assert_almost_equal(fine,Q*coarse) - assert_almost_equal(Q*(Q.T*fine),fine) + assert_almost_equal(fine_candidates,Q*coarse_candidates) + assert_almost_equal(Q*(Q.T*fine_candidates),fine_candidates) - #aggregate one more level (to a single aggregate) - K = len(coarse_candidates) - N = K*AggOp.shape[1] - AggOp = csr_matrix((ones(N),zeros(N),arange(N + 1)),dims=(N,1)) #aggregate to a single point - fine_candidates = coarse_candidates - - #mask_candidate(AggOp,fine_candidates) #not needed - - #now check the coarser problem - Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) +## #aggregate one more level (to a single aggregate) +## K = coarse_candidates.shape[1] +## N = K*AggOp.shape[1] +## AggOp = csr_matrix((ones(N),zeros(N),arange(N + 1)),dims=(N,1)) #aggregate to a single point +## fine_candidates = coarse_candidates +## +## #mask_candidate(AggOp,fine_candidates) #not needed +## +## #now check the coarser problem +## Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) +## +## assert_almost_equal(fine_candidates,Q*coarse_candidates) +## assert_almost_equal(Q*(Q.T*fine_candidates),fine_candidates) - assert_equal(len(coarse_candidates),len(fine_candidates)) - for fine,coarse in zip(fine_candidates,coarse_candidates): - assert_almost_equal(fine,Q*coarse) - assert_almost_equal(Q*(Q.T*fine),fine) - - - class TestSASolverPerformance(NumpyTestCase): def setUp(self): self.cases = [] @@ -223,9 +215,9 @@ DAD = D*A*D if candidates is None: - candidates = [ ones(A.shape[0]) ] + candidates = ones((A.shape[0],1)) - DAD_candidates = [ (D_inv * c) for c in candidates ] + DAD_candidates = D_inv * candidates #TODO force 2 level method and check that result is the same Modified: trunk/scipy/sandbox/multigrid/tests/test_utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_utils.py 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/tests/test_utils.py 2007-10-22 01:18:47 UTC (rev 3452) @@ -2,12 +2,13 @@ import numpy import scipy -from scipy import matrix,array,diag +from scipy import matrix,array,diag,zeros from scipy.sparse import csr_matrix set_package_path() -from scipy.sandbox.multigrid.utils import infinity_norm,diag_sparse +from scipy.sandbox.multigrid.utils import infinity_norm, diag_sparse, \ + expand_into_blocks restore_path() @@ -52,9 +53,29 @@ A = matrix([[1.3,0,0],[0,5.5,0],[0,0,-2]]) assert_equal(diag_sparse(array([1.3,5.5,-2])).todense(),csr_matrix(A).todense()) + def check_expand_into_blocks(self): + cases = [] + cases.append( ( matrix([[1]]), (1,2) ) ) + cases.append( ( matrix([[1]]), (2,1) ) ) + cases.append( ( matrix([[1]]), (2,2) ) ) + cases.append( ( matrix([[1,2]]), (1,2) ) ) + cases.append( ( matrix([[1,2],[3,4]]), (2,2) ) ) + cases.append( ( matrix([[0,0],[0,0]]), (3,1) ) ) + cases.append( ( matrix([[0,1,0],[0,2,3]]), (3,2) ) ) + cases.append( ( matrix([[1,0,0],[2,0,3]]), (2,5) ) ) + for A,dims in cases: + m,n = dims + result = expand_into_blocks(csr_matrix(A),m,n).todense() + expected = zeros((m*A.shape[0],n*A.shape[1])) + for i in range(m): + for j in range(n): + expected[i::m,j::n] = A + assert_equal(expected,result) + + if __name__ == '__main__': NumpyTest().run() Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-22 00:21:14 UTC (rev 3451) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-22 01:18:47 UTC (rev 3452) @@ -3,22 +3,21 @@ import numpy import scipy -from numpy import ravel,arange,concatenate,tile +from numpy import ravel,arange,concatenate,tile,asarray from scipy.linalg import norm from scipy.sparse import isspmatrix,isspmatrix_csr,isspmatrix_csc, \ csr_matrix,csc_matrix,extract_diagonal, \ coo_matrix -def approximate_spectral_radius(A,tol=0.1,maxiter=20): +def approximate_spectral_radius(A,tol=0.1,maxiter=None): """ Approximate the spectral radius of a symmetric matrix using ARPACK """ from scipy.sandbox.arpack import eigen - return norm(eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False)[0]) + return norm(eigen(A, k=1, ncv=10, which='LM', maxiter=maxiter, tol=tol, return_eigenvectors=False)) - def infinity_norm(A): """ Infinity norm of a sparse matrix (maximum absolute row sum). This serves @@ -44,12 +43,16 @@ if isspmatrix(A): return extract_diagonal(A) else: - return csr_matrix((A,arange(len(A)),arange(len(A)+1)),(len(A),len(A))) + return csr_matrix((asarray(A),arange(len(A)),arange(len(A)+1)),(len(A),len(A))) def hstack_csr(A,B): - #TODO OPTIMIZE THIS - assert(A.shape[0] == B.shape[0]) + if not isspmatrix(A) or not isspmatrix(B): + raise TypeError,'expected sparse matrix' + + if A.shape[0] != B.shape[0]: + raise ValueError,'row dimensions must agree' + A = A.tocoo() B = B.tocoo() I = concatenate((A.row,B.row)) @@ -59,7 +62,12 @@ def vstack_csr(A,B): #TODO OPTIMIZE THIS - assert(A.shape[1] == B.shape[1]) + if not isspmatrix(A) or not isspmatrix(B): + raise TypeError,'expected sparse matrix' + + if A.shape[0] != B.shape[0]: + raise ValueError,'row dimensions must agree' + A = A.tocoo() B = B.tocoo() I = concatenate((A.row,B.row+A.shape[0])) @@ -84,6 +92,7 @@ """ #TODO EXPLAIN MORE + #TODO use spkron instead, time for compairson if n is None: n = m From scipy-svn at scipy.org Mon Oct 22 08:10:28 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 22 Oct 2007 07:10:28 -0500 (CDT) Subject: [Scipy-svn] r3453 - trunk/scipy/ndimage Message-ID: <20071022121028.34AEC39C10D@new.scipy.org> Author: stefan Date: 2007-10-22 07:10:00 -0500 (Mon, 22 Oct 2007) New Revision: 3453 Modified: trunk/scipy/ndimage/_ni_support.py trunk/scipy/ndimage/interpolation.py Log: Fix zooming. Modified: trunk/scipy/ndimage/_ni_support.py =================================================================== --- trunk/scipy/ndimage/_ni_support.py 2007-10-22 01:18:47 UTC (rev 3452) +++ trunk/scipy/ndimage/_ni_support.py 2007-10-22 12:10:00 UTC (rev 3453) @@ -65,7 +65,7 @@ import warnings def _get_output(output, input, output_type = None, shape = None): if output_type != None: - msg = "'output_type' argument is decrepated." + msg = "'output_type' argument is deprecated." msg += " Assign type to 'output' instead." raise RuntimeError, msg warnings.warn(msg, DeprecationWarning) Modified: trunk/scipy/ndimage/interpolation.py =================================================================== --- trunk/scipy/ndimage/interpolation.py 2007-10-22 01:18:47 UTC (rev 3452) +++ trunk/scipy/ndimage/interpolation.py 2007-10-22 12:10:00 UTC (rev 3453) @@ -37,9 +37,6 @@ def _extend_mode_to_code(mode): mode = _ni_support._extend_mode_to_code(mode) - if mode == 2: - warnings.warn('Mode "reflect" may yield incorrect results on ' - 'boundaries. Please use "mirror" instead.') return mode def spline_filter1d(input, order = 3, axis = -1, output = numpy.float64, @@ -324,12 +321,11 @@ filtered = input zoom = _ni_support._normalize_sequence(zoom, input.ndim) output_shape = [int(ii * jj) for ii, jj in zip(input.shape, zoom)] - zoom = [1.0 / ii for ii in zoom] + zoom = (numpy.array(input.shape)-1)/(numpy.array(output_shape,float)-1) output, return_value = _ni_support._get_output(output, input, output_type, shape = output_shape) zoom = numpy.asarray(zoom, dtype = numpy.float64) - if not zoom.flags.contiguous: - zoom = shift.copy() + zoom = numpy.ascontiguousarray(zoom) _nd_image.zoom_shift(filtered, zoom, None, output, order, mode, cval) return return_value From scipy-svn at scipy.org Mon Oct 22 08:23:35 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 22 Oct 2007 07:23:35 -0500 (CDT) Subject: [Scipy-svn] r3454 - trunk/scipy/ndimage/tests Message-ID: <20071022122335.27FC939C1A7@new.scipy.org> Author: stefan Date: 2007-10-22 07:23:21 -0500 (Mon, 22 Oct 2007) New Revision: 3454 Modified: trunk/scipy/ndimage/tests/test_ndimage.py Log: New tests for zoom. Modified: trunk/scipy/ndimage/tests/test_ndimage.py =================================================================== --- trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-22 12:10:00 UTC (rev 3453) +++ trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-22 12:23:21 UTC (rev 3454) @@ -2114,91 +2114,23 @@ [0, 4, 1, 3], [0, 7, 6, 8]]) < eps) - def test_zoom01(self): + def test_zoom1(self): "zoom 1" - data = numpy.ones([2], numpy.float64) - for order in range(0, 6): - out = ndimage.zoom(data, 2.0, order=order) - self.failUnless(diff(out, [1, 1, 1, 0]) < eps) + for order in range(0,6): + for z in [2,[2,2]]: + arr = numpy.array(range(25)).reshape((5,5)).astype(float) + arr = ndimage.zoom(arr, z, order=2) + assert_equal(arr.shape,(10,10)) + assert numpy.all(arr[-1,:] != 0) + assert numpy.all(arr[-1,:] >= 20) + assert numpy.all(arr[0,:] <= 5) - def test_zoom02(self): + def test_zoom2(self): "zoom 2" - data = [1, 5, 2, 6, 3, 7, 4, 4] - for order in range(0, 6): - out = ndimage.zoom(data, 0.5, order=order) - self.failUnless(diff(out, [1, 2, 3, 4]) < eps) + arr = numpy.arange(12).reshape((3,4)) + out = ndimage.zoom(ndimage.zoom(arr,2),0.5) + assert_array_equal(out,arr) - def test_zoom03(self): - "zoom 3" - data = [1, 2, 3, 4] - for order in range(0, 6): - out = ndimage.zoom(data, 2, order=order) - self.failUnless(diff(out[::2], [1, 2, 3, 4]) < eps) - - def test_zoom04(self): - "zoom 4" - data = [[1, 2, 3, 4], - [5, 6, 7, 8], - [9.0, 10, 11, 12]] - for order in range(0, 6): - out = ndimage.zoom(data, [1, 0.5], order=order) - self.failUnless(diff(out, [[1, 3], [5, 7], [9, 11]]) < eps) - - def test_zoom05(self): - "zoom 5" - data = [[1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12]] - for order in range(0, 6): - out = ndimage.zoom(data, [0.5, 1], order=order) - self.failUnless(diff(out, [[1, 2, 3, 4]]) < eps) - - def test_zoom06(self): - "zoom 6" - data = [[1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12]] - for order in range(0, 6): - out = ndimage.zoom(data, [0.5, 0.5], order=order) - self.failUnless(diff(out, [[1, 3]]) < eps) - - def test_zoom07(self): - "zoom 7" - data = [[1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12]] - for order in range(0, 6): - out = ndimage.zoom(data, [1, 2], order=order) - self.failUnless(diff(out[..., ::2], data) < eps) - - def test_zoom08(self): - "zoom 8" - data = [[1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12]] - for order in range(0, 6): - out = ndimage.zoom(data, [2, 1], order=order) - self.failUnless(diff(out[::2, ...], data) < eps) - - def test_zoom09(self): - "zoom 9" - data = [[1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12]] - for order in range(0, 6): - out = ndimage.zoom(data, [2, 2], order=order) - self.failUnless(diff(out[::2, ::2], data) < eps) - - def test_zoom10(self): - "zoom 10" - data = numpy.array([[1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12]], numpy.float64) - for order in range(0, 6): - out = ndimage.zoom(data, [2, 2], order=order) - out = ndimage.zoom(out, [0.5, 0.5], order=order) - self.failUnless(diff(out, data) < eps) - def test_zoom_affine01(self): "zoom by affine transformation 1" data = [[1, 2, 3, 4], From scipy-svn at scipy.org Mon Oct 22 08:29:18 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 22 Oct 2007 07:29:18 -0500 (CDT) Subject: [Scipy-svn] r3455 - trunk/scipy/ndimage/tests Message-ID: <20071022122918.D6E8639C1A7@new.scipy.org> Author: stefan Date: 2007-10-22 07:28:58 -0500 (Mon, 22 Oct 2007) New Revision: 3455 Modified: trunk/scipy/ndimage/tests/test_ndimage.py Log: Check range of zoom values. Modified: trunk/scipy/ndimage/tests/test_ndimage.py =================================================================== --- trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-22 12:23:21 UTC (rev 3454) +++ trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-22 12:28:58 UTC (rev 3455) @@ -2124,6 +2124,8 @@ assert numpy.all(arr[-1,:] != 0) assert numpy.all(arr[-1,:] >= 20) assert numpy.all(arr[0,:] <= 5) + assert numpy.all(arr >= 0) + assert numpy.all(arr <= 24) def test_zoom2(self): "zoom 2" From scipy-svn at scipy.org Mon Oct 22 12:13:25 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 22 Oct 2007 11:13:25 -0500 (CDT) Subject: [Scipy-svn] r3456 - trunk/scipy/ndimage/tests Message-ID: <20071022161325.511B9C7C025@new.scipy.org> Author: stefan Date: 2007-10-22 11:13:03 -0500 (Mon, 22 Oct 2007) New Revision: 3456 Modified: trunk/scipy/ndimage/tests/test_ndimage.py Log: Protect against floating point roundoff. Modified: trunk/scipy/ndimage/tests/test_ndimage.py =================================================================== --- trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-22 12:28:58 UTC (rev 3455) +++ trunk/scipy/ndimage/tests/test_ndimage.py 2007-10-22 16:13:03 UTC (rev 3456) @@ -2119,13 +2119,13 @@ for order in range(0,6): for z in [2,[2,2]]: arr = numpy.array(range(25)).reshape((5,5)).astype(float) - arr = ndimage.zoom(arr, z, order=2) + arr = ndimage.zoom(arr, z, order=order) assert_equal(arr.shape,(10,10)) assert numpy.all(arr[-1,:] != 0) - assert numpy.all(arr[-1,:] >= 20) - assert numpy.all(arr[0,:] <= 5) - assert numpy.all(arr >= 0) - assert numpy.all(arr <= 24) + assert numpy.all(arr[-1,:] >= (20 - eps)) + assert numpy.all(arr[0,:] <= (5 + eps)) + assert numpy.all(arr >= (0 - eps)) + assert numpy.all(arr <= (24 + eps)) def test_zoom2(self): "zoom 2" From scipy-svn at scipy.org Mon Oct 22 16:13:08 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 22 Oct 2007 15:13:08 -0500 (CDT) Subject: [Scipy-svn] r3457 - trunk/scipy/sandbox/multigrid Message-ID: <20071022201308.6C71539C118@new.scipy.org> Author: wnbell Date: 2007-10-22 15:13:01 -0500 (Mon, 22 Oct 2007) New Revision: 3457 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/relaxation.py Log: updated aSA initialization step to new candidate representation Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-22 16:13:03 UTC (rev 3456) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-22 20:13:01 UTC (rev 3457) @@ -1,5 +1,6 @@ import numpy,scipy,scipy.sparse -from numpy import sqrt,ravel,diff,zeros,zeros_like,inner,concatenate,asarray +from numpy import sqrt, ravel, diff, zeros, zeros_like, inner, concatenate, \ + asarray, hstack from scipy.sparse import csr_matrix,coo_matrix from relaxation import gauss_seidel @@ -116,16 +117,16 @@ Ws = AggOps - self.candidates = [x] + self.candidates = x #create SA using x here - As,Is,Ps, = sa_hierarchy(A,AggOps,self.candidates) + As,Is,Ps = sa_hierarchy(A,AggOps,self.candidates) for i in range(max_candidates - 1): #x = self.__develop_candidate(A,As,Is,Ps,Ws,AggOps,mu=mu) x = self.__develop_candidate(As,Is,Ps,AggOps,self.candidates,mu=mu) - self.candidates.append(x) + self.candidates = hstack((self.candidates,x)) #TODO which is faster? #As,Is,Ps,Ws = self.__augment_cycle(A,As,Ps,Ws,AggOps,x) @@ -146,12 +147,11 @@ #step 1 A_l = A - x = scipy.rand(A_l.shape[0]) + x = scipy.rand(A_l.shape[0],1) skip_f_to_i = False #step 2 - b = zeros_like(x) - gauss_seidel(A_l,x,b,iterations=mu,sweep='symmetric') + gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #step 3 #TODO test convergence rate here @@ -160,11 +160,11 @@ while len(AggOps) + 1 < max_levels and A_l.shape[0] > max_coarse: W_l = sa_constant_interpolation(A_l,epsilon=0,blocks=blocks) #step 4b - P_l,x = sa_fit_candidates(W_l,[x]) #step 4c - x = x[0] #TODO make sa_fit_candidates accept a single x + P_l,x = sa_fit_candidates(W_l,x) #step 4c I_l = smoothed_prolongator(P_l,A_l) #step 4d A_l = I_l.T.tocsr() * A_l * I_l #step 4e - + + print 'A_l.shape',A_l.shape AggOps.append(W_l) Is.append(I_l) As.append(A_l) @@ -175,8 +175,8 @@ print "." x_hat = x.copy() #step 4g gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #step 4h - x_A_x = inner(x,A_l*x) - if (x_A_x/inner(x_hat,A_l*x_hat))**(1.0/mu) < epsilon: #step 4i + x_A_x = dot(x.T,A_l*x) + if (x_A_x/dot(x_hat.T,A_l*x_hat))**(1.0/mu) < epsilon: #step 4i print "sufficient convergence, skipping" skip_f_to_i = True if x_A_x == 0: @@ -186,7 +186,7 @@ for A_l,I in reversed(zip(As[1:],Is)): gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST x = I * x - gauss_seidel(A,x,b,iterations=mu) #TEST + gauss_seidel(A,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST return x,AggOps #first candidate,aggregation @@ -315,7 +315,7 @@ blocks = None -A = poisson_problem2D(200) +A = poisson_problem2D(100) #A = io.mmread("tests/sample_data/laplacian_41_3dcube.mtx").tocsr() #A = io.mmread("laplacian_40_3dcube.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() @@ -325,10 +325,10 @@ #D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() #A = D * A * D -A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() -blocks = arange(A.shape[0]/2).repeat(2) +#A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() +#blocks = arange(A.shape[0]/2).repeat(2) -asa = adaptive_sa_solver(A,max_candidates=4,mu=10) +asa = adaptive_sa_solver(A,max_candidates=1,mu=10) scipy.random.seed(0) #make tests repeatable x = rand(A.shape[0]) #b = A*rand(A.shape[0]) @@ -356,7 +356,7 @@ print "constant Rayleigh quotient",dot(ones(A.shape[0]),A*ones(A.shape[0]))/float(A.shape[0]) -for c in asa.candidates: +for c in asa.candidates.T: print "candidate Rayleigh quotient",dot(c,A*c)/dot(c,c) Modified: trunk/scipy/sandbox/multigrid/relaxation.py =================================================================== --- trunk/scipy/sandbox/multigrid/relaxation.py 2007-10-22 16:13:03 UTC (rev 3456) +++ trunk/scipy/sandbox/multigrid/relaxation.py 2007-10-22 20:13:01 UTC (rev 3457) @@ -1,5 +1,5 @@ import multigridtools -from numpy import empty_like +from numpy import empty_like, asarray def sor(A,x,b,omega,iterations=1,sweep='forward'): @@ -30,6 +30,9 @@ sweep - direction of sweep: 'forward' (default), 'backward', or 'symmetric' """ + x = asarray(x).reshape(-1) + b = asarray(b).reshape(-1) + if A.shape[0] != A.shape[1]: raise ValueError,'expected symmetric matrix' From scipy-svn at scipy.org Tue Oct 23 00:08:26 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Mon, 22 Oct 2007 23:08:26 -0500 (CDT) Subject: [Scipy-svn] r3458 - trunk/scipy/sandbox/multigrid Message-ID: <20071023040826.35EA839C018@new.scipy.org> Author: wnbell Date: 2007-10-22 23:08:10 -0500 (Mon, 22 Oct 2007) New Revision: 3458 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/relaxation.py trunk/scipy/sandbox/multigrid/utils.py Log: continued work on adaptiveSA Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-22 20:13:01 UTC (rev 3457) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-23 04:08:10 UTC (rev 3458) @@ -6,12 +6,66 @@ from relaxation import gauss_seidel from multilevel import multilevel_solver from sa import sa_constant_interpolation,sa_fit_candidates -from utils import approximate_spectral_radius,hstack_csr,vstack_csr +from utils import approximate_spectral_radius,hstack_csr,vstack_csr,expand_into_blocks + +def augment_candidates(AggOp, old_Q, old_R, new_candidate, level): + K = old_R.shape[1] + #determine blocksizes + if level == 0: + old_bs = (1,K) + new_bs = (1,K+1) + else: + old_bs = (K,K) + new_bs = (K+1,K+1) + + AggOp = expand_into_blocks(AggOp,new_bs[0],1).tocsr() #TODO switch to block matrix + # tentative prolongator + #TODO USE BSR + Q_indptr = (K+1)*AggOp.indptr + Q_indices = ((K+1)*AggOp.indices).repeat(K+1) + for i in range(K+1): + Q_indices[i::K+1] += i + Q_data = zeros((AggOp.shape[0]/new_bs[0],) + new_bs) + Q_data[:,:new_bs[0],:new_bs[1]] = old_Q.data.reshape((-1,) + old_bs) #TODO BSR change + # coarse candidates + R = zeros(((K+1)*AggOp.shape[1],K+1)) + for i in range(K): + R[i::K+1,:K] = old_R[i::K,:] + + c = new_candidate.reshape(-1)[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses + X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) + + #orthogonalize X against previous + for i in range(K): + old_c = ascontiguousarray(Q_data[:,:,i].reshape(-1)) + D_AtX = csr_matrix((old_c*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X + R[i::K+1,-1] = D_AtX + X.data -= D_AtX[X.indices] * old_c + + #normalize X + D_XtX = csr_matrix((X.data**2,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of X.T * X + col_norms = sqrt(D_XtX) + R[K::K+1,-1] = col_norms + + col_norms = 1.0/col_norms + col_norms[isinf(col_norms)] = 0 + X.data *= col_norms[X.indices] + last_column = Q_data[:,:,-1].reshape(-1) + last_column = X.data.reshape(-1) + Q_data = Q_data.reshape(-1) #TODO BSR change + + Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(AggOp.shape[0],(K+1)*AggOp.shape[1])) + + return Q,R + + + + def orthonormalize_prolongator(P_l,x_l,W_l,W_m): """ @@ -72,26 +126,28 @@ Construct multilevel hierarchy using Smoothed Aggregation Inputs: A - matrix - Is - list of constant prolongators + Ps - list of constant prolongators B - "candidate" basis function to be approximated Ouputs: - (As,Is,Ps) - tuple of lists - - As - [A, Ps[0].T*A*Ps[0], Ps[1].T*A*Ps[1], ... ] - - Is - smoothed prolongators - - Ps - tentative prolongators + (As,Ps,Ts) - tuple of lists + - As - [A, Ts[0].T*A*Ts[0], Ts[1].T*A*Ts[1], ... ] + - Ps - smoothed prolongators + - Ts - tentative prolongators """ As = [A] - Is = [] Ps = [] + Ts = [] + Bs = [B] for W in Ws: P,B = sa_fit_candidates(W,B) I = smoothed_prolongator(P,A) A = I.T.tocsr() * A * I As.append(A) - Ps.append(P) - Is.append(I) - return As,Is,Ps + Ts.append(P) + Ps.append(I) + Bs.append(B) + return As,Ps,Ts,Bs def make_bridge(I,N): tail = I.indptr[-1].repeat(N - I.shape[0]) @@ -117,29 +173,30 @@ Ws = AggOps - self.candidates = x + fine_candidates = x #create SA using x here - As,Is,Ps = sa_hierarchy(A,AggOps,self.candidates) + As,Ps,Ts,self.candidates = sa_hierarchy(A,AggOps,fine_candidates) + #self.candidates = [x] for i in range(max_candidates - 1): - #x = self.__develop_candidate(A,As,Is,Ps,Ws,AggOps,mu=mu) - x = self.__develop_candidate(As,Is,Ps,AggOps,self.candidates,mu=mu) + #x = self.__develop_candidate_OLD(A,As,Ps,Ts,Ws,AggOps,mu=mu) + #As,Ps,Ts,Ws = self.__augment_cycle(A,As,Ts,Ws,AggOps,x) + #self.candidates.append(x) - self.candidates = hstack((self.candidates,x)) - #TODO which is faster? - #As,Is,Ps,Ws = self.__augment_cycle(A,As,Ps,Ws,AggOps,x) - As,Is,Ps = sa_hierarchy(A,AggOps,self.candidates) + x = self.__develop_candidate(As,Ps,Ts,AggOps,self.candidates,mu=mu) + fine_candidates = hstack((fine_candidates,x)) + As,Ps,Ts,self.candidates = sa_hierarchy(A,AggOps,fine_candidates) - self.Ps = Ps - self.solver = multilevel_solver(As,Is) + self.Ts = Ts + self.solver = multilevel_solver(As,Ps) self.AggOps = AggOps def __initialization_stage(self,A,blocks,max_levels,max_coarse,mu,epsilon): AggOps = [] - Is = [] + Ps = [] # aSA parameters # mu - number of test relaxation iterations @@ -166,7 +223,7 @@ print 'A_l.shape',A_l.shape AggOps.append(W_l) - Is.append(I_l) + Ps.append(I_l) As.append(A_l) if A_l.shape <= max_coarse: break @@ -183,7 +240,7 @@ x = x_hat #need to restore x #update fine-level candidate - for A_l,I in reversed(zip(As[1:],Is)): + for A_l,I in reversed(zip(As[1:],Ps)): gauss_seidel(A_l,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST x = I * x gauss_seidel(A,x,zeros_like(x),iterations=mu,sweep='symmetric') #TEST @@ -191,93 +248,113 @@ return x,AggOps #first candidate,aggregation - def __develop_candidate(self,As,Is,Ps,AggOps,candidates,mu): - #scipy.random.seed(0) #TEST - x = scipy.rand(A.shape[0]) + def __develop_candidate(self,As,Ps,Ts,AggOps,candidates,mu): + A = As[0] + + x = A*scipy.rand(A.shape[0],1) b = zeros_like(x) - solver = multilevel_solver(As,Is) - - x = solver.solve(b, x0=x, tol=1e-10, maxiter=mu) - + x = multilevel_solver(As,Ps).solve(b, x0=x, tol=1e-10, maxiter=mu) + #TEST FOR CONVERGENCE HERE - #TODO augment candiates each time, then call fit_candidates? + temp_Ps = [] + temp_As = [A] + + def make_bridge(P,K): + indptr = P.indptr[:-1].reshape(-1,K-1) + indptr = hstack((indptr,indptr[:,-1].reshape(-1,1))) + indptr = indptr.reshape(-1) + indptr = hstack((indptr,indptr[-1:])) #duplicate last element + return csr_matrix((P.data,P.indices,indptr),dims=(K*P.shape[0]/(K-1),P.shape[1])) - A_l,P_l,W_l,x_l = As[0],Ps[0],Ws[0],x - - temp_Is = [] - for i in range(len(As) - 2): - P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, AggOps[i+1]) + for i in range(len(As) - 2): + #TODO test augment_candidates against fit candidates + if i == 0: + temp = hstack((candidates[0],x)) + else: + K = candidates[i].shape[1] + temp = zeros((x.shape[0]/(K+1),K+1,K + 1)) + temp[:,:-1,:-1] = candidates[i].reshape(-1,K,K) + temp[:,:,-1] = x.reshape(-1,K+1,1) + temp = temp.reshape(-1,K+1) + T_,R_ = sa_fit_candidates(AggOps[i],temp) + #print "T - T_",(T - T_).data.max() + #assert((T - T_).data.max() < 1e-10) + #assert((R - R_).data.max() < 1e-10) + T,R = T_,R_ + #TODO end test + + #T,R = augment_candidates(AggOps[i], Ts[i], candidates[i+1], x, i) + P = smoothed_prolongator(T,A) + A = P.T.tocsr() * A * P + + temp_Ps.append(P) + temp_As.append(A) + + #TODO USE BSR (K,K) -> (K,K-1) + bridge = make_bridge(Ps[i+1],R.shape[1]) - I_l_new = smoothed_prolongator(P_l_new,A_l) - A_m_new = I_l_new.T.tocsr() * A_l * I_l_new - bridge = make_bridge(Is[i+1],A_m_new.shape[0]) + solver = multilevel_solver( [A] + As[i+2:], [bridge] + Ps[i+2:] ) + + x = R[:,-1].reshape(-1,1) + x = solver.solve(zeros_like(x), x0=x, tol=1e-8, maxiter=mu) - temp_solver = multilevel_solver( [A_m_new] + As[i+2:], [bridge] + Is[i+2:] ) - - for n in range(mu): - x_m = temp_solver.solve(zeros_like(x_m), x0=x_m, tol=1e-8, maxiter=1) - - temp_Is.append(I_l_new) - - W_l = vstack_csr(Ws[i+1],W_m_new) #prepare for next iteration - A_l = A_m_new - x_l = x_m - P_l = make_bridge(Ps[i+1],A_m_new.shape[0]) - - x = x_l - for I in reversed(temp_Is): - x = I*x + for A,P in reversed(zip(temp_As,temp_Ps)): + x = P * x + gauss_seidel(A,x,zeros_like(x),iterations=mu,sweep='symmetric') + #for P in reversed(temp_Ps): + # x = P*x + return x -## def __develop_candidate(self,A,As,Is,Ps,Ws,AggOps,mu): +## def __develop_candidate_OLD(self,A,As,Ps,Ts,Ws,AggOps,mu): ## #scipy.random.seed(0) #TEST ## x = scipy.rand(A.shape[0]) ## b = zeros_like(x) ## -## solver = multilevel_solver(As,Is) +## solver = multilevel_solver(As,Ps) ## ## x = solver.solve(b, x0=x, tol=1e-10, maxiter=mu) ## ## #TEST FOR CONVERGENCE HERE ## -## A_l,P_l,W_l,x_l = As[0],Ps[0],Ws[0],x +## A_l,P_l,W_l,x_l = As[0],Ts[0],Ws[0],x ## -## temp_Is = [] +## temp_Ps = [] ## for i in range(len(As) - 2): ## P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, AggOps[i+1]) ## ## I_l_new = smoothed_prolongator(P_l_new,A_l) ## A_m_new = I_l_new.T.tocsr() * A_l * I_l_new -## bridge = make_bridge(Is[i+1],A_m_new.shape[0]) +## bridge = make_bridge(Ps[i+1],A_m_new.shape[0]) ## -## temp_solver = multilevel_solver( [A_m_new] + As[i+2:], [bridge] + Is[i+2:] ) +## temp_solver = multilevel_solver( [A_m_new] + As[i+2:], [bridge] + Ps[i+2:] ) ## ## for n in range(mu): ## x_m = temp_solver.solve(zeros_like(x_m), x0=x_m, tol=1e-8, maxiter=1) ## -## temp_Is.append(I_l_new) +## temp_Ps.append(I_l_new) ## ## W_l = vstack_csr(Ws[i+1],W_m_new) #prepare for next iteration ## A_l = A_m_new ## x_l = x_m -## P_l = make_bridge(Ps[i+1],A_m_new.shape[0]) +## P_l = make_bridge(Ts[i+1],A_m_new.shape[0]) ## ## x = x_l -## for I in reversed(temp_Is): +## for I in reversed(temp_Ps): ## x = I*x ## ## return x - def __augment_cycle(self,A,As,Ps,Ws,AggOps,x): + def __augment_cycle(self,A,As,Ts,Ws,AggOps,x): #make a new cycle using the new candidate - A_l,P_l,W_l,x_l = As[0],Ps[0],AggOps[0],x + A_l,P_l,W_l,x_l = As[0],Ts[0],AggOps[0],x - new_As,new_Is,new_Ps,new_Ws = [A],[],[],[AggOps[0]] + new_As,new_Ps,new_Ts,new_Ws = [A],[],[],[AggOps[0]] for i in range(len(As) - 2): P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, AggOps[i+1]) @@ -288,24 +365,24 @@ new_As.append(A_m_new) new_Ws.append(W_m_new) - new_Is.append(I_l_new) - new_Ps.append(P_l_new) + new_Ps.append(I_l_new) + new_Ts.append(P_l_new) #prepare for next iteration W_l = W_m_new A_l = A_m_new x_l = x_m - P_l = make_bridge(Ps[i+1],A_m_new.shape[0]) + P_l = make_bridge(Ts[i+1],A_m_new.shape[0]) P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, csr_matrix((P_l.shape[1],1))) I_l_new = smoothed_prolongator(P_l_new,A_l) A_m_new = I_l_new.T.tocsr() * A_l * I_l_new new_As.append(A_m_new) - new_Is.append(I_l_new) - new_Ps.append(P_l_new) + new_Ps.append(I_l_new) + new_Ts.append(P_l_new) - return new_As,new_Is,new_Ps,new_Ws + return new_As,new_Ps,new_Ts,new_Ws @@ -325,10 +402,10 @@ #D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() #A = D * A * D -#A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() -#blocks = arange(A.shape[0]/2).repeat(2) +A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() +blocks = arange(A.shape[0]/2).repeat(2) -asa = adaptive_sa_solver(A,max_candidates=1,mu=10) +asa = adaptive_sa_solver(A,max_candidates=4,mu=5) scipy.random.seed(0) #make tests repeatable x = rand(A.shape[0]) #b = A*rand(A.shape[0]) @@ -356,7 +433,7 @@ print "constant Rayleigh quotient",dot(ones(A.shape[0]),A*ones(A.shape[0]))/float(A.shape[0]) -for c in asa.candidates.T: +for c in asa.candidates[0].T: print "candidate Rayleigh quotient",dot(c,A*c)/dot(c,c) @@ -364,6 +441,19 @@ ##W = asa.AggOps[0]*asa.AggOps[1] ##pcolor((W * rand(W.shape[1])).reshape((200,200))) +def plot2d_arrows(x): + from pylab import quiver + x = x.reshape(-1) + N = (len(x)/2)**0.5 + assert(2 * N * N == len(x)) + X = linspace(-1,1,N).reshape(1,N).repeat(N,0).reshape(-1) + Y = linspace(-1,1,N).reshape(1,N).repeat(N,0).T.reshape(-1) + + dX = x[0::2] + dY = x[1::2] + + quiver(X,Y,dX,dY) + def plot2d(x): from pylab import pcolor pcolor(x.reshape(sqrt(len(x)),sqrt(len(x)))) Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-22 20:13:01 UTC (rev 3457) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-23 04:08:10 UTC (rev 3458) @@ -168,7 +168,7 @@ if x0 is None: x = zeros_like(b) else: - x = array(x0) + x = array(x0) #copy #TODO change use of tol (relative tolerance) to agree with other iterative solvers @@ -193,19 +193,21 @@ A = self.As[lvl] if len(self.As) == 1: - x[:] = spsolve(A,b) + #TODO make spsolve preserve dimensions + x[:] = spsolve(A,b).reshape(x.shape) return self.presmoother(A,x,b) residual = b - A*x - coarse_x = zeros((self.As[lvl+1].shape[0])) coarse_b = self.Ps[lvl].T * residual + coarse_x = zeros_like(coarse_b) if lvl == len(self.As) - 2: #use direct solver on coarsest level - coarse_x[:] = spsolve(self.As[-1],coarse_b) #TODO reuse factors for efficiency? + #TODO reuse factors for efficiency? + coarse_x[:] = spsolve(self.As[-1],coarse_b).reshape(coarse_x.shape) #coarse_x[:] = scipy.linalg.cg(self.As[-1],coarse_b,tol=1e-12)[0] #print "coarse residual norm",scipy.linalg.norm(coarse_b - self.As[-1]*coarse_x) else: Modified: trunk/scipy/sandbox/multigrid/relaxation.py =================================================================== --- trunk/scipy/sandbox/multigrid/relaxation.py 2007-10-22 20:13:01 UTC (rev 3457) +++ trunk/scipy/sandbox/multigrid/relaxation.py 2007-10-23 04:08:10 UTC (rev 3458) @@ -77,6 +77,9 @@ iterations - number of iterations to perform (default: 1) omega - damping parameter (default: 1.0) """ + x = asarray(x).reshape(-1) + b = asarray(b).reshape(-1) + sweep = slice(None) (row_start,row_stop,row_step) = sweep.indices(A.shape[0]) Modified: trunk/scipy/sandbox/multigrid/utils.py =================================================================== --- trunk/scipy/sandbox/multigrid/utils.py 2007-10-22 20:13:01 UTC (rev 3457) +++ trunk/scipy/sandbox/multigrid/utils.py 2007-10-23 04:08:10 UTC (rev 3458) @@ -65,8 +65,8 @@ if not isspmatrix(A) or not isspmatrix(B): raise TypeError,'expected sparse matrix' - if A.shape[0] != B.shape[0]: - raise ValueError,'row dimensions must agree' + if A.shape[1] != B.shape[1]: + raise ValueError,'column dimensions must agree' A = A.tocoo() B = B.tocoo() @@ -94,9 +94,6 @@ #TODO EXPLAIN MORE #TODO use spkron instead, time for compairson - if n is None: - n = m - if m == 1 and n == 1: return A #nothing to do From scipy-svn at scipy.org Wed Oct 24 00:20:15 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 23 Oct 2007 23:20:15 -0500 (CDT) Subject: [Scipy-svn] r3459 - in trunk/scipy/sandbox/multigrid: . tests Message-ID: <20071024042015.1A80339C1B0@new.scipy.org> Author: wnbell Date: 2007-10-23 23:19:51 -0500 (Tue, 23 Oct 2007) New Revision: 3459 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/tests/test_adaptive.py trunk/scipy/sandbox/multigrid/tests/test_sa.py Log: aSA works reasonably well now Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-23 04:08:10 UTC (rev 3458) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-24 04:19:51 UTC (rev 3459) @@ -1,6 +1,7 @@ import numpy,scipy,scipy.sparse from numpy import sqrt, ravel, diff, zeros, zeros_like, inner, concatenate, \ - asarray, hstack + asarray, hstack, ascontiguousarray, isinf +from numpy.random import randn from scipy.sparse import csr_matrix,coo_matrix from relaxation import gauss_seidel @@ -10,11 +11,12 @@ -def augment_candidates(AggOp, old_Q, old_R, new_candidate, level): +def augment_candidates(AggOp, old_Q, old_R, new_candidate): K = old_R.shape[1] #determine blocksizes - if level == 0: + if new_candidate.shape[0] == old_Q.shape[0]: + #then this is the first prolongator old_bs = (1,K) new_bs = (1,K+1) else: @@ -22,42 +24,51 @@ new_bs = (K+1,K+1) AggOp = expand_into_blocks(AggOp,new_bs[0],1).tocsr() #TODO switch to block matrix - + + # tentative prolongator #TODO USE BSR Q_indptr = (K+1)*AggOp.indptr Q_indices = ((K+1)*AggOp.indices).repeat(K+1) for i in range(K+1): Q_indices[i::K+1] += i - Q_data = zeros((AggOp.shape[0]/new_bs[0],) + new_bs) - Q_data[:,:new_bs[0],:new_bs[1]] = old_Q.data.reshape((-1,) + old_bs) #TODO BSR change + Q_data = zeros((AggOp.indptr[-1]/new_bs[0],) + new_bs) + Q_data[:,:old_bs[0],:old_bs[1]] = old_Q.data.reshape((-1,) + old_bs) #TODO BSR change # coarse candidates - R = zeros(((K+1)*AggOp.shape[1],K+1)) - for i in range(K): - R[i::K+1,:K] = old_R[i::K,:] + #R = zeros(((K+1)*AggOp.shape[1],K+1)) + #for i in range(K): + # R[i::K+1,:K] = old_R[i::K,:] + R = zeros((AggOp.shape[1],K+1,K+1)) + R[:,:K,:K] = old_R.reshape(-1,K,K) c = new_candidate.reshape(-1)[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses + threshold = 1e-10 / abs(c).max() # cutoff for small basis functions + X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) #orthogonalize X against previous for i in range(K): old_c = ascontiguousarray(Q_data[:,:,i].reshape(-1)) D_AtX = csr_matrix((old_c*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X - R[i::K+1,-1] = D_AtX + R[:,i,K] = D_AtX X.data -= D_AtX[X.indices] * old_c #normalize X D_XtX = csr_matrix((X.data**2,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of X.T * X col_norms = sqrt(D_XtX) - R[K::K+1,-1] = col_norms - + mask = col_norms < threshold # find small basis functions + col_norms[mask] = 0 # and set them to zero + + R[:,K,K] = col_norms # store diagonal entry into R + col_norms = 1.0/col_norms - col_norms[isinf(col_norms)] = 0 + col_norms[mask] = 0 X.data *= col_norms[X.indices] - last_column = Q_data[:,:,-1].reshape(-1) - last_column = X.data.reshape(-1) + Q_data[:,:,-1] = X.data.reshape(-1,1) + Q_data = Q_data.reshape(-1) #TODO BSR change + R = R.reshape(-1,K+1) Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(AggOp.shape[0],(K+1)*AggOp.shape[1])) @@ -171,8 +182,8 @@ max_coarse = max_coarse, \ mu = mu, epsilon = epsilon) - Ws = AggOps - + #Ws = AggOps + x /= abs(x).max() fine_candidates = x #create SA using x here @@ -186,6 +197,7 @@ #TODO which is faster? x = self.__develop_candidate(As,Ps,Ts,AggOps,self.candidates,mu=mu) + x /= abs(x).max() fine_candidates = hstack((fine_candidates,x)) As,Ps,Ts,self.candidates = sa_hierarchy(A,AggOps,fine_candidates) @@ -204,7 +216,7 @@ #step 1 A_l = A - x = scipy.rand(A_l.shape[0],1) + x = randn(A_l.shape[0],1) skip_f_to_i = False #step 2 @@ -251,7 +263,7 @@ def __develop_candidate(self,As,Ps,Ts,AggOps,candidates,mu): A = As[0] - x = A*scipy.rand(A.shape[0],1) + x = randn(A.shape[0],1) b = zeros_like(x) x = multilevel_solver(As,Ps).solve(b, x0=x, tol=1e-10, maxiter=mu) @@ -270,22 +282,22 @@ for i in range(len(As) - 2): #TODO test augment_candidates against fit candidates - if i == 0: - temp = hstack((candidates[0],x)) - else: - K = candidates[i].shape[1] - temp = zeros((x.shape[0]/(K+1),K+1,K + 1)) - temp[:,:-1,:-1] = candidates[i].reshape(-1,K,K) - temp[:,:,-1] = x.reshape(-1,K+1,1) - temp = temp.reshape(-1,K+1) - T_,R_ = sa_fit_candidates(AggOps[i],temp) - #print "T - T_",(T - T_).data.max() - #assert((T - T_).data.max() < 1e-10) - #assert((R - R_).data.max() < 1e-10) - T,R = T_,R_ + T,R = augment_candidates(AggOps[i], Ts[i], candidates[i+1], x) + #if i == 0: + # temp = hstack((candidates[0],x)) + #else: + # K = candidates[i].shape[1] + # temp = zeros((x.shape[0]/(K+1),K+1,K + 1)) + # temp[:,:-1,:-1] = candidates[i].reshape(-1,K,K) + # temp[:,:,-1] = x.reshape(-1,K+1,1) + # temp = temp.reshape(-1,K+1) + #T_,R_ = sa_fit_candidates(AggOps[i],temp) + #print "T - T_",abs(T - T_).sum() + #assert(abs(T - T_).sum() < 1e-10) + #assert((R - R_).sum() < 1e-10) + #T,R = T_,R_ #TODO end test - #T,R = augment_candidates(AggOps[i], Ts[i], candidates[i+1], x, i) P = smoothed_prolongator(T,A) A = P.T.tocsr() * A * P @@ -385,76 +397,83 @@ return new_As,new_Ps,new_Ts,new_Ws +if __name__ == '__main__': + from scipy import * + from utils import diag_sparse + from multilevel import poisson_problem1D,poisson_problem2D -from scipy import * -from utils import diag_sparse -from multilevel import poisson_problem1D,poisson_problem2D + blocks = None + + A = poisson_problem2D(100) + #A = io.mmread("tests/sample_data/laplacian_41_3dcube.mtx").tocsr() + #A = io.mmread("laplacian_40_3dcube.mtx").tocsr() + #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() + #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() + + + #D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() + #A = D * A * D + + A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() + blocks = arange(A.shape[0]/2).repeat(2) + + asa = adaptive_sa_solver(A,max_candidates=3,mu=10) + #scipy.random.seed(0) #make tests repeatable + x = randn(A.shape[0]) + b = A*randn(A.shape[0]) + #b = zeros(A.shape[0]) + -blocks = None - -A = poisson_problem2D(100) -#A = io.mmread("tests/sample_data/laplacian_41_3dcube.mtx").tocsr() -#A = io.mmread("laplacian_40_3dcube.mtx").tocsr() -#A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() -#A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() - - -#D = diag_sparse(1.0/sqrt(10**(12*rand(A.shape[0])-6))).tocsr() -#A = D * A * D - -A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() -blocks = arange(A.shape[0]/2).repeat(2) - -asa = adaptive_sa_solver(A,max_candidates=4,mu=5) -scipy.random.seed(0) #make tests repeatable -x = rand(A.shape[0]) -#b = A*rand(A.shape[0]) -b = zeros(A.shape[0]) - - -print "solving" -if True: - x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=25,tol=1e-7,return_residuals=True) -else: - residuals = [] - def add_resid(x): - residuals.append(linalg.norm(b - A*x)) - A.psolve = asa.solver.psolve - x_sol = linalg.cg(A,b,x0=x,maxiter=20,tol=1e-12,callback=add_resid)[0] - -residuals = array(residuals)/residuals[0] - -print "residuals ",residuals -print "mean convergence factor",(residuals[-1]/residuals[0])**(1.0/len(residuals)) -print "last convergence factor",residuals[-1]/residuals[-2] - -print -print asa.solver - -print "constant Rayleigh quotient",dot(ones(A.shape[0]),A*ones(A.shape[0]))/float(A.shape[0]) - -for c in asa.candidates[0].T: - print "candidate Rayleigh quotient",dot(c,A*c)/dot(c,c) - - - -##W = asa.AggOps[0]*asa.AggOps[1] -##pcolor((W * rand(W.shape[1])).reshape((200,200))) - -def plot2d_arrows(x): - from pylab import quiver - x = x.reshape(-1) - N = (len(x)/2)**0.5 - assert(2 * N * N == len(x)) - X = linspace(-1,1,N).reshape(1,N).repeat(N,0).reshape(-1) - Y = linspace(-1,1,N).reshape(1,N).repeat(N,0).T.reshape(-1) - - dX = x[0::2] - dY = x[1::2] - - quiver(X,Y,dX,dY) - -def plot2d(x): - from pylab import pcolor - pcolor(x.reshape(sqrt(len(x)),sqrt(len(x)))) + print "solving" + if True: + x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=20,tol=1e-12,return_residuals=True) + else: + residuals = [] + def add_resid(x): + residuals.append(linalg.norm(b - A*x)) + A.psolve = asa.solver.psolve + x_sol = linalg.cg(A,b,x0=x,maxiter=30,tol=1e-12,callback=add_resid)[0] + residuals = array(residuals)/residuals[0] + + print "residuals ",residuals + print "mean convergence factor",(residuals[-1]/residuals[0])**(1.0/len(residuals)) + print "last convergence factor",residuals[-1]/residuals[-2] + + print + print asa.solver + + print "constant Rayleigh quotient",dot(ones(A.shape[0]),A*ones(A.shape[0]))/float(A.shape[0]) + + + def plot2d_arrows(x): + from pylab import figure,quiver,show + x = x.reshape(-1) + N = (len(x)/2)**0.5 + assert(2 * N * N == len(x)) + X = linspace(-1,1,N).reshape(1,N).repeat(N,0).reshape(-1) + Y = linspace(-1,1,N).reshape(1,N).repeat(N,0).T.reshape(-1) + + dX = x[0::2] + dY = x[1::2] + + figure() + quiver(X,Y,dX,dY) + show() + + def plot2d(x): + from pylab import pcolor,figure,show + figure() + pcolor(x.reshape(sqrt(len(x)),sqrt(len(x)))) + show() + + for c in asa.candidates[0].T: + plot2d_arrows(c) + print "candidate Rayleigh quotient",dot(c,A*c)/dot(c,c) + + + + ##W = asa.AggOps[0]*asa.AggOps[1] + ##pcolor((W * rand(W.shape[1])).reshape((200,200))) + + Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-23 04:08:10 UTC (rev 3458) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-24 04:19:51 UTC (rev 3459) @@ -4,7 +4,7 @@ import scipy import numpy -from numpy import ones,zeros,zeros_like,array +from numpy import ones,zeros,zeros_like,array,asarray from numpy.linalg import norm from scipy.linsolve import spsolve Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-23 04:08:10 UTC (rev 3458) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-24 04:19:51 UTC (rev 3459) @@ -104,7 +104,7 @@ def sa_fit_candidates(AggOp,candidates): K = candidates.shape[1] # num candidates - + N_fine,N_coarse = AggOp.shape if K > 1 and candidates.shape[0] == K*N_fine: @@ -112,27 +112,32 @@ AggOp = expand_into_blocks(AggOp,K,1).tocsr() N_fine = K*N_fine - R = zeros((K*N_coarse,K)) #storage for coarse candidates + R = zeros((N_coarse,K,K)) #storage for coarse candidates candidate_matrices = [] for i in range(K): c = candidates[:,i] - c = c[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses + c = c[diff(AggOp.indptr) == 1] # eliminate DOFs that aggregation misses + + threshold = 1e-10 / abs(c).max() # cutoff for small basis functions + X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) #orthogonalize X against previous for j,A in enumerate(candidate_matrices): D_AtX = csr_matrix((A.data*X.data,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of A.T * X - R[j::K,i] = D_AtX + R[:,j,i] = D_AtX X.data -= D_AtX[X.indices] * A.data #normalize X D_XtX = csr_matrix((X.data**2,X.indices,X.indptr),dims=X.shape).sum(axis=0).A.flatten() #same as diagonal of X.T * X col_norms = sqrt(D_XtX) - R[i::K,i] = col_norms + mask = col_norms < threshold # set small basis functions to 0 + col_norms[mask] = 0 + R[:,i,i] = col_norms col_norms = 1.0/col_norms - col_norms[isinf(col_norms)] = 0 + col_norms[mask] = 0 X.data *= col_norms[X.indices] candidate_matrices.append(X) @@ -147,6 +152,8 @@ Q_data[i::K] = X.data Q = csr_matrix((Q_data,Q_indices,Q_indptr),dims=(N_fine,K*N_coarse)) + R = R.reshape(-1,K) + return Q,R def sa_smoothed_prolongator(A,T,epsilon,omega,blocks=None): Modified: trunk/scipy/sandbox/multigrid/tests/test_adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_adaptive.py 2007-10-23 04:08:10 UTC (rev 3458) +++ trunk/scipy/sandbox/multigrid/tests/test_adaptive.py 2007-10-24 04:19:51 UTC (rev 3459) @@ -1,17 +1,70 @@ from numpy.testing import * from scipy.sparse import csr_matrix -from scipy import arange,ones,zeros,array,eye +from numpy import arange,ones,zeros,array,eye,vstack,diff set_package_path() -pass +from scipy.sandbox.multigrid.sa import sa_fit_candidates +from scipy.sandbox.multigrid.adaptive import augment_candidates restore_path() +#import pdb; pdb.set_trace() class TestAdaptiveSA(NumpyTestCase): def setUp(self): pass + +class TestAugmentCandidates(NumpyTestCase): + def setUp(self): + self.cases = [] + + #two candidates + + ##block candidates + ##self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((array([1]*9 + [0]*9),arange(2*9))).T )) + + + def check_first_level(self): + cases = [] + + ## tests where AggOp includes all DOFs + cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)), vstack((ones(4),arange(4))).T )) + cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((ones(9),arange(9))).T )) + cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)), vstack((ones(9),arange(9))).T )) + + ## tests where AggOp excludes some DOFs + cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5))).T )) + + # overdetermined blocks + cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5),arange(5)**2)).T )) + cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9),arange(9)**2)).T )) + cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9))).T )) + + def mask_candidate(AggOp,candidates): + #mask out all DOFs that are not included in the aggregation + candidates[diff(AggOp.indptr) == 0,:] = 0 + + for AggOp,fine_candidates in cases: + + mask_candidate(AggOp,fine_candidates) + + for i in range(1,fine_candidates.shape[1]): + Q_expected,R_expected = sa_fit_candidates(AggOp,fine_candidates[:,:i+1]) + + old_Q, old_R = sa_fit_candidates(AggOp,fine_candidates[:,:i]) + + Q_result,R_result = augment_candidates(AggOp, old_Q, old_R, fine_candidates[:,[i]]) + + # compare against SA method (which is assumed to be correct) + assert_almost_equal(Q_expected.todense(),Q_result.todense()) + assert_almost_equal(R_expected,R_result) + + #each fine level candidate should be fit exactly + assert_almost_equal(fine_candidates[:,:i+1],Q_result*R_result) + assert_almost_equal(Q_result*(Q_result.T*fine_candidates[:,:i+1]),fine_candidates[:,:i+1]) + + if __name__ == '__main__': NumpyTest().run() Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-23 04:08:10 UTC (rev 3458) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-24 04:19:51 UTC (rev 3459) @@ -143,6 +143,10 @@ ## tests where AggOp excludes some DOFs self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), ones((5,1)) )) self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5))).T )) + + # overdetermined blocks + self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5),arange(5)**2)).T )) + self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9),arange(9)**2)).T )) self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9))).T )) def check_all_cases(self): @@ -153,7 +157,6 @@ candidates[diff(AggOp.indptr) == 0,:] = 0 for AggOp,fine_candidates in self.cases: - mask_candidate(AggOp,fine_candidates) Q,coarse_candidates = sa_fit_candidates(AggOp,fine_candidates) From scipy-svn at scipy.org Wed Oct 24 17:20:25 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 24 Oct 2007 16:20:25 -0500 (CDT) Subject: [Scipy-svn] r3460 - in trunk/scipy/sparse: . tests Message-ID: <20071024212025.4EAEF39C20E@new.scipy.org> Author: wnbell Date: 2007-10-24 16:20:22 -0500 (Wed, 24 Oct 2007) New Revision: 3460 Modified: trunk/scipy/sparse/sparse.py trunk/scipy/sparse/tests/test_sparse.py Log: add support for dense -> coo_matrix conversion Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-24 04:19:51 UTC (rev 3459) +++ trunk/scipy/sparse/sparse.py 2007-10-24 21:20:22 UTC (rev 3460) @@ -536,7 +536,6 @@ def _imag(self): return self._with_data(numpy.imag(self.data)) - def _binopt(self, other, fn, in_shape=None, out_shape=None): """apply the binary operation fn to two sparse matrices""" other = self._tothis(other) @@ -2150,10 +2149,12 @@ COO matrices are created either as: A = coo_matrix(None, dims=(m, n), [dtype]) for a zero matrix, or as: + A = coo_matrix(M) + where M is a dense matrix or rank 2 ndarray, or as: A = coo_matrix((obj, ij), [dims]) where the dimensions are optional. If supplied, we set (M, N) = dims. - If not supplied, we infer these from the index arrays - ij[0][:] and ij[1][:] + If not supplied, we infer these from the index arrays: + ij[0][:] and ij[1][:] The arguments 'obj' and 'ij' represent three arrays: 1. obj[:] the entries of the matrix, in any order @@ -2162,6 +2163,12 @@ So the following holds: A[ij[0][k], ij[1][k] = obj[k] + + Note: + When converting to CSR or CSC format, duplicate (i,j) entries + will be summed together. This facilitates efficient construction + of finite element matrices and the like. + """ def __init__(self, arg1, dims=None, dtype=None): spmatrix.__init__(self) @@ -2170,6 +2177,29 @@ obj, ij = arg1 except: raise TypeError, "invalid input format" + + try: + if len(ij) != 2: + raise TypeError + except TypeError: + raise TypeError, "invalid input format" + + self.row = asarray(ij[0], dtype=numpy.intc) + self.col = asarray(ij[1], dtype=numpy.intc) + self.data = asarray(obj) + self.dtype = self.data.dtype + + if dims is None: + if len(self.row) == 0 or len(self.col) == 0: + raise ValueError, "cannot infer dimensions from zero sized index arrays" + M = self.row.max() + 1 + N = self.col.max() + 1 + self.shape = (M, N) + else: + # Use 2 steps to ensure dims has length 2. + M, N = dims + self.shape = (M, N) + elif arg1 is None: # clumsy! We should make ALL arguments # keyword arguments instead! # Initialize an empty matrix. @@ -2180,34 +2210,22 @@ self.data = array([]) self.row = array([]) self.col = array([]) - self._check() - return else: - raise TypeError, "invalid input format" + #dense argument + try: + M = asarray(arg1) + except: + raise TypeError, "invalid input format" - self.dtype = getdtype(dtype, obj, default=float) + if len(M.shape) != 2: + raise TypeError, "expected rank 2 array or matrix" + self.shape = M.shape + self.dtype = M.dtype + self.row,self.col = (M != 0).nonzero() + self.data = M[self.row,self.col] - try: - if len(ij) != 2: - raise TypeError - except TypeError: - raise TypeError, "invalid input format" - - if dims is None: - if len(ij[0]) == 0 or len(ij[1]) == 0: - raise ValueError, "cannot infer dimensions from zero sized index arrays" - M = int(amax(ij[0])) + 1 - N = int(amax(ij[1])) + 1 - self.shape = (M, N) - else: - # Use 2 steps to ensure dims has length 2. - M, N = dims - self.shape = (M, N) - - self.row = asarray(ij[0], dtype=numpy.intc) - self.col = asarray(ij[1], dtype=numpy.intc) - self.data = asarray(obj, dtype=self.dtype) self._check() + def _check(self): """ Checks for consistency and stores the number of non-zeros as @@ -2236,25 +2254,27 @@ self.shape = tuple([int(x) for x in self.shape]) self.nnz = nnz - def _normalize(self, rowfirst=False): - if rowfirst: - #sort by increasing rows first, columns second - if getattr(self, '_is_normalized', None): - #columns already sorted, use stable sort for rows - P = numpy.argsort(self.row, kind='mergesort') - return self.data[P], self.row[P], self.col[P] - else: - #nothing already sorted - P = numpy.lexsort(keys=(self.col, self.row)) - return self.data[P], self.row[P], self.col[P] - if getattr(self, '_is_normalized', None): - return self.data, self.row, self.col - #sort by increasing rows first, columns second - P = numpy.lexsort(keys=(self.row, self.col)) - self.data, self.row, self.col = self.data[P], self.row[P], self.col[P] - setattr(self, '_is_normalized', 1) - return self.data, self.row, self.col +## _normalize shouldn't be necessary anymore +## def _normalize(self, rowfirst=False): +## if rowfirst: +## #sort by increasing rows first, columns second +## if getattr(self, '_is_normalized', None): +## #columns already sorted, use stable sort for rows +## P = numpy.argsort(self.row, kind='mergesort') +## return self.data[P], self.row[P], self.col[P] +## else: +## #nothing already sorted +## P = numpy.lexsort(keys=(self.col, self.row)) +## return self.data[P], self.row[P], self.col[P] +## if getattr(self, '_is_normalized', None): +## return self.data, self.row, self.col +## #sort by increasing rows first, columns second +## P = numpy.lexsort(keys=(self.row, self.col)) +## self.data, self.row, self.col = self.data[P], self.row[P], self.col[P] +## setattr(self, '_is_normalized', 1) +## return self.data, self.row, self.col + def rowcol(self, num): return (self.row[num], self.col[num]) @@ -2266,7 +2286,7 @@ return csc_matrix(self.shape, dtype=self.dtype) else: indptr, rowind, data = cootocsc(self.shape[0], self.shape[1], \ - self.size, self.row, self.col, \ + self.nnz, self.row, self.col, \ self.data) return csc_matrix((data, rowind, indptr), self.shape, check=False) @@ -2276,7 +2296,7 @@ return csr_matrix(self.shape, dtype=self.dtype) else: indptr, colind, data = cootocsr(self.shape[0], self.shape[1], \ - self.size, self.row, self.col, \ + self.nnz, self.row, self.col, \ self.data) return csr_matrix((data, colind, indptr), self.shape, check=False) Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-24 04:19:51 UTC (rev 3459) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-24 21:20:22 UTC (rev 3460) @@ -1111,6 +1111,7 @@ class TestCOO(NumpyTestCase): def check_constructor1(self): + """unsorted triplet format""" row = numpy.array([2, 3, 1, 3, 0, 1, 3, 0, 2, 1, 2]) col = numpy.array([0, 1, 0, 0, 1, 1, 2, 2, 2, 2, 1]) data = numpy.array([ 6., 10., 3., 9., 1., 4., @@ -1121,7 +1122,7 @@ assert_array_equal(arange(12).reshape(4,3),coo.todense()) def check_constructor2(self): - #duplicate entries should be summed + """unsorted triplet format with duplicates (which are summed)""" row = numpy.array([0,1,2,2,2,2,0,0,2,2]) col = numpy.array([0,2,0,2,1,1,1,0,0,2]) data = numpy.array([2,9,-4,5,7,0,-1,2,1,-5]) @@ -1131,38 +1132,51 @@ assert_array_equal(mat,coo.todense()) - def check_normalize( self ): + def check_constructor3(self): + """empty matrix""" + coo = coo_matrix(None,dims=(4,3)) - row = numpy.array([2, 3, 1, 3, 0, 1, 3, 0, 2, 1, 2]) - col = numpy.array([0, 1, 0, 0, 1, 1, 2, 2, 2, 2, 1]) - data = numpy.array([ 6., 10., 3., 9., 1., 4., - 11., 2., 8., 5., 7.]) + assert_array_equal(zeros((4,3)),coo.todense()) - # coo.todense() - # matrix([[ 0., 1., 2.], - # [ 3., 4., 5.], - # [ 6., 7., 8.], - # [ 9., 10., 11.]]) - coo = coo_matrix((data,(row,col)),(4,3)) + def check_constructor4(self): + """from dense matrix""" + mat = numpy.array([[0,1,0,0], + [7,0,3,0], + [0,4,0,0]]) + coo = coo_matrix(mat) + assert_array_equal(mat,coo.todense()) + +## def check_normalize( self ): +## row = numpy.array([2, 3, 1, 3, 0, 1, 3, 0, 2, 1, 2]) +## col = numpy.array([0, 1, 0, 0, 1, 1, 2, 2, 2, 2, 1]) +## data = numpy.array([ 6., 10., 3., 9., 1., 4., +## 11., 2., 8., 5., 7.]) +## +## # coo.todense() +## # matrix([[ 0., 1., 2.], +## # [ 3., 4., 5.], +## # [ 6., 7., 8.], +## # [ 9., 10., 11.]]) +## coo = coo_matrix((data,(row,col)),(4,3)) +## +## ndata,nrow,ncol = coo._normalize(rowfirst=True) +## sorted_rcd = zip(row, col, data) +## sorted_rcd.sort() +## assert(zip(nrow,ncol,ndata) == sorted_rcd) #should sort by rows, then cols +## assert_array_equal(coo.data, data) #coo.data has not changed +## assert_array_equal(coo.row, row) #coo.row has not changed +## assert_array_equal(coo.col, col) #coo.col has not changed +## +## +## ndata,nrow,ncol = coo._normalize(rowfirst=False) +## assert(zip(ncol,nrow,ndata) == sorted(zip(col,row,data))) #should sort by cols, then rows +## assert_array_equal(coo.data, ndata) #coo.data has changed +## assert_array_equal(coo.row, nrow) #coo.row has changed +## assert_array_equal(coo.col, ncol) #coo.col has changed +## +## assert_array_equal(coo.tocsr().todense(), coo.todense()) +## assert_array_equal(coo.tocsc().todense(), coo.todense()) - ndata,nrow,ncol = coo._normalize(rowfirst=True) - sorted_rcd = zip(row, col, data) - sorted_rcd.sort() - assert(zip(nrow,ncol,ndata) == sorted_rcd) #should sort by rows, then cols - assert_array_equal(coo.data, data) #coo.data has not changed - assert_array_equal(coo.row, row) #coo.row has not changed - assert_array_equal(coo.col, col) #coo.col has not changed - - ndata,nrow,ncol = coo._normalize(rowfirst=False) - assert(zip(ncol,nrow,ndata) == sorted(zip(col,row,data))) #should sort by cols, then rows - assert_array_equal(coo.data, ndata) #coo.data has changed - assert_array_equal(coo.row, nrow) #coo.row has changed - assert_array_equal(coo.col, ncol) #coo.col has changed - - assert_array_equal(coo.tocsr().todense(), coo.todense()) - assert_array_equal(coo.tocsc().todense(), coo.todense()) - - if __name__ == "__main__": NumpyTest().run() From scipy-svn at scipy.org Wed Oct 24 18:55:21 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 24 Oct 2007 17:55:21 -0500 (CDT) Subject: [Scipy-svn] r3461 - in trunk/scipy/io: . tests Message-ID: <20071024225521.A34B439C0BF@new.scipy.org> Author: wnbell Date: 2007-10-24 17:55:18 -0500 (Wed, 24 Oct 2007) New Revision: 3461 Modified: trunk/scipy/io/mmio.py trunk/scipy/io/tests/test_mmio.py Log: simplified and improved speed of matrix market sparse IO added complex sparse MMIO unittest Modified: trunk/scipy/io/mmio.py =================================================================== --- trunk/scipy/io/mmio.py 2007-10-24 21:20:22 UTC (rev 3460) +++ trunk/scipy/io/mmio.py 2007-10-24 22:55:18 UTC (rev 3461) @@ -14,6 +14,7 @@ import os from numpy import asarray, real, imag, conj, zeros, ndarray +from itertools import izip __all__ = ['mminfo','mmread','mmwrite'] @@ -334,12 +335,16 @@ format = '%i %i ' + format target.write('%i %i %i\n' % (rows,cols,entries)) assert symm=='general',`symm` + + coo = a.tocoo() # convert to COOrdinate format + I,J,V = coo.row + 1, coo.col + 1, coo.data # change base 0 -> base 1 + if field in ['real','integer']: - for i in range(entries): - target.write(format % (a.rowcol(i)[0] + 1,a.rowcol(i)[1] + 1,a.getdata(i))) #convert base 0 to base 1 + for ijv_tuple in izip(I,J,V): + target.writelines(format % ijv_tuple) elif field=='complex': - for i in range(entries): - target.write(format % (a.rowcol(i)[0] + 1,a.rowcol(i)[1] + 1,real(a.getdata(i)),imag(a.getdata(i)))) + for ijv_tuple in izip(I,J,V.real,V.imag): + target.writelines(format % ijv_tuple) elif field=='pattern': raise NotImplementedError,`field` else: Modified: trunk/scipy/io/tests/test_mmio.py =================================================================== --- trunk/scipy/io/tests/test_mmio.py 2007-10-24 21:20:22 UTC (rev 3460) +++ trunk/scipy/io/tests/test_mmio.py 2007-10-24 22:55:18 UTC (rev 3461) @@ -152,7 +152,7 @@ b = mmread(fn).todense() assert_array_almost_equal(a,b) - def check_simple_write_read(self): + def check_real_write_read(self): I = array([0, 0, 1, 2, 3, 3, 3, 4]) J = array([0, 3, 1, 2, 1, 3, 4, 4]) V = array([ 1.0, 6.0, 10.5, 0.015, 250.5, -280.0, 33.32, 12.0 ]) @@ -163,14 +163,25 @@ mmwrite(fn,b) assert_equal(mminfo(fn),(5,5,8,'coordinate','real','general')) - a = [[1, 0, 0, 6, 0], - [0, 10.5, 0, 0, 0], - [0, 0, .015, 0, 0], - [0, 250.5, 0, -280, 33.32], - [0, 0, 0, 0, 12]] + a = b.todense() b = mmread(fn).todense() assert_array_almost_equal(a,b) + def check_complex_write_read(self): + I = array([0, 0, 1, 2, 3, 3, 3, 4]) + J = array([0, 3, 1, 2, 1, 3, 4, 4]) + V = array([ 1.0 + 3j, 6.0 + 2j, 10.50 + 0.9j, 0.015 + -4.4j, + 250.5 + 0j, -280.0 + 5j, 33.32 + 6.4j, 12.00 + 0.8j]) + + b = scipy.sparse.coo_matrix((V,(I,J)),dims=(5,5)) + + fn = mktemp() + mmwrite(fn,b) + + assert_equal(mminfo(fn),(5,5,8,'coordinate','complex','general')) + a = b.todense() + b = mmread(fn).todense() + assert_array_almost_equal(a,b) if __name__ == "__main__": NumpyTest().run() From scipy-svn at scipy.org Thu Oct 25 13:09:57 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 25 Oct 2007 12:09:57 -0500 (CDT) Subject: [Scipy-svn] r3462 - trunk/scipy/weave/doc Message-ID: <20071025170957.145A739C115@new.scipy.org> Author: cookedm Date: 2007-10-25 12:09:53 -0500 (Thu, 25 Oct 2007) New Revision: 3462 Added: trunk/scipy/weave/doc/tutorial.txt trunk/scipy/weave/doc/tutorial_original.html Removed: trunk/scipy/weave/doc/tutorial.html Log: Convert weave/doc/tutorial.html to ReST syntax. - Move original to tutorial_original.html. Should be removed at a later date. - Add tutorial.txt - Some minor updates (sorry, lost track of them). However, it's still out-of-date. Deleted: trunk/scipy/weave/doc/tutorial.html =================================================================== --- trunk/scipy/weave/doc/tutorial.html 2007-10-24 22:55:18 UTC (rev 3461) +++ trunk/scipy/weave/doc/tutorial.html 2007-10-25 17:09:53 UTC (rev 3462) @@ -1,2753 +0,0 @@ - - - - - -

Weave Documentation

-

-By Eric Jones eric at enthought.com -

-

-

Outline

-
-
Introduction
-
Requirements
-
Installation
-
Testing
-
Benchmarks
-
Inline -
-
More with printf
-
More examples -
-
Binary search
-
Dictionary sort
-
NumPy -- -cast/copy/transpose
-
wxPython
-
-
-
Keyword options
-
Returning values -
-
The issue -with locals()
-
-
-
A quick look at the code -
-
Technical Details -
-
Converting Types -
-
NumPy -Argument Conversion
-
String, -List, Tuple, and Dictionary Conversion
-
File -Conversion
-
Callable, -Instance, and Module Conversion
-
Customizing -Conversions
-
-
-
Compiling Code
-
"Cataloging" functions -
-
Function Storage
-
The PYTHONCOMPILED -evnironment variable
-
-
-
-
-
-
-
Blitz -
-
Requirements
-
Limitations
-
NumPy Efficiency Issues
-
The Tools -
-
Parser
-
Blitz and NumPy
-
-
-
Type defintions and coersion -
-
Cataloging Compiled Functions
-
Checking Array Sizes
-
Creating the Extension -Module
-
-
-
Extension Modules -
-
A Simple Example
-
Fibonacci Example
-
-
-
Customizing Type Conversions -- -Type Factories (not written) -
-
Type Specifications
-
Type Information
-
The Conversion Process
-
-
-
- -

Introduction

-

-The weave package provides tools for including C/C++ code -within -in Python code. This offers both another level of optimization to those -who need it, and an easy way to modify and extend any supported -extension libraries such as wxPython and hopefully VTK soon. Inlining -C/C++ code within Python generally -results in speed ups of 1.5x to 30x speed-up over algorithms written in -pure -Python (However, it is also possible to slow things down...). Generally -algorithms that require a large number of calls to the Python API don't -benefit -as much from the conversion to C/C++ as algorithms that have inner -loops completely convertable to C. -

-

There are three basic ways to use weave. The weave.inline() -function executes C code directly within Python, and weave.blitz() -translates Python NumPy expressions to C++ for fast execution. blitz() -was the original reason weave was built. For those -interested in building extension -libraries, the ext_tools module provides classes for -building extension modules within Python.

-

Most of weave's functionality should work on Windows -and Unix, although some of its functionality requires gcc -or a similarly modern C++ compiler that handles templates well. Up to -now, most testing has been done on Windows 2000 with Microsoft's C++ -compiler (MSVC) and with gcc (mingw32 2.95.2 and 2.95.3-6). All tests -also pass on Linux (RH 7.1 with gcc 2.96), and I've had reports that it -works on Debian also (thanks Pearu). -

-

The inline and blitz provide new -functionality to Python (although I've recently learned about the PyInline project which may -offer similar functionality to inline). On the other -hand, tools for building Python extension modules already exists (SWIG, -SIP, pycpp, CXX, and others). As of yet, I'm not sure where weave -fits in this spectrum. It is closest in flavor to CXX in that it makes -creating new C/C++ extension modules pretty easy. However, if you're -wrapping a gaggle of legacy functions or classes, SWIG and friends are -definitely the better choice. weave is set up so that you -can customize how Python types are converted to C types in weave. -This is great for inline(), but, for wrapping legacy -code, it is more flexible to specify things the other way around -- -that is how C types map to Python types. This weave does -not do. I guess it would be possible to build such a tool on top of weave, -but with good tools like SWIG around, I'm not sure the effort produces -any new capabilities. Things like function overloading are probably -easily implemented in weave and it might be easier to mix -Python/C code in function calls, but nothing beyond this comes to mind. -So, if you're developing new extension modules or optimizing Python -functions in C, weave.ext_tools() might be the tool for -you. If you're wrapping legacy code, stick with SWIG. -

-

The next several sections give the basics of how to use weave. -We'll discuss what's happening under the covers in more detail later -on. Serious users will need to at least look at the type conversion -section to understand how Python variables map to C/C++ types and how -to customize this behavior. One other note. If you don't know C or C++ -then these docs are probably of very little help to you. Further, it'd -be helpful if you know something about writing Python extensions. weave -does quite a bit for you, but for anything complex, you'll need to do -some conversions, reference counting, etc. -

-

-Note: weave is actually part of the SciPy package. However, it also works -fine as a standalone package (you can check out the sources using svn -co http://svn.scipy.org/svn/scipy/trunk/Lib/weave weave and install as -python setup.py install). The examples here are given as if it is used -as a stand alone package. If you are using from within scipy, you can -use from scipy import weave and the examples will work -identically. -

-

Requirements

-
    -
  • Python -

    I use 2.1.1. Probably 2.0 or higher should work.

    -

    -
    • -
  • C++ compiler -

    weave uses distutils to actually -build extension modules, so it uses whatever compiler was originally -used to build Python. weave itself requires a C++ -compiler. If you used a C++ compiler to build Python, your probably -fine.

    -

    On Unix gcc is the preferred choice because I've done a little -testing with it. All testing has been done with gcc, but I expect the -majority of compilers should work for inline and ext_tools. -The one issue I'm not sure about is that I've hard coded things so that -compilations are linked with the stdc++ library. Is -this standard across Unix compilers, or is this a gcc-ism?

    -

    For blitz(), you'll need a reasonably recent -version of gcc. 2.95.2 works on windows and 2.96 looks fine on Linux. -Other versions are likely to work. Its likely that KAI's C++ compiler -and maybe some others will work, but I haven't tried. My advice is to -use gcc for now unless your willing to tinker with the code some.

    -

    On Windows, either MSVC or gcc ( mingw32) should work. -Again, you'll need gcc for blitz() as the MSVC compiler -doesn't handle templates well.

    -

    I have not tried Cygwin, so please report success if it works -for you.

    -

    -
    • -
  • NumPy -

    The python NumPy module from here. -is required for blitz() to work and for numpy.distutils -which is used by weave.

    -

    -
    • -
-

-

-

Installation

-

-There are currently two ways to get weave. Fist, weave -is part of SciPy and installed automatically (as a sub- -package) whenever SciPy is installed. Second, since weave -is useful outside of the scientific community, it has been setup so -that it can be -used as a stand-alone module.

-

The stand-alone version can be downloaded from here.  Instructions for -installing should be found there as well.  setup.py file to -simplify -installation.

-

-

Testing

-Once weave is installed, fire up python and run its unit -tests. -
- 

-    >>> import weave
-    >>> weave.test()
-    runs long time... spews tons of output and a few warnings
-    .
-    .
-    .
-    ..............................................................
-    ................................................................
-    ..................................................
-    ----------------------------------------------------------------------
-    Ran 184 tests in 158.418s
-
-    OK
-    
-    >>> 
-
-
-This takes a while, usually several minutes. -On Unix with remote file systems, I've had it take 15 or so minutes. In -the end, it should run about 180 tests and spew some speed results -along the way. If you get errors, they'll be reported at the end of the -output. Please report erros that you find.   Some tests are -known to fail at this point.
-

If you only want to test a single module of the package, you can do -this by -running test() for that specific module.

-
- 

-    >>> import weave.scalar_spec
-    >>> weave.scalar_spec.test()
-    .......
-    ----------------------------------------------------------------------
-    Ran 7 tests in 23.284s
-
-
-Testing Notes: - -
    - -
  • Windows 1 -

    I've had some test fail on windows machines where I have msvc, -gcc-2.95.2 (in c:\gcc-2.95.2), and gcc-2.95.3-6 (in c:\gcc) all -installed. My environment has c:\gcc in the path and does not have -c:\gcc-2.95.2 in the path. The test process runs very smoothly until -the end where several test using gcc fail with cpp0 not found by g++. -If I check os.system('gcc -v') before running tests, I get -gcc-2.95.3-6. If I check after running tests (and after failure), I get -gcc-2.95.2. ??huh??. The os.environ['PATH'] still has c:\gcc first in -it and is not corrupted (msvc/distutils messes with the environment -variables, so we have to undo its work in some places). If anyone else -sees this, let me know - - it may just be an quirk on my machine -(unlikely). Testing with the gcc- 2.95.2 installation always works.

    -

    -
    • - -
  • Windows 2 -

    If you run the tests from PythonWin or some other GUI tool, -you'll get a ton of DOS windows popping up periodically as weave -spawns the compiler multiple times. Very annoying. Anyone know how to -fix this?

    -

    -
    • - -
  • wxPython -

    wxPython tests are not enabled by default because importing -wxPython on a Unix machine without access to a X-term will cause the -program to exit. Anyone know of a safe way to detect whether wxPython -can be imported and whether a display exists on a machine?

    -

    -

    -
    • - -
- -

Benchmarks

-This section has not been updated from old scipy weave and Numeric....
-
-This section has a few benchmarks  -- thats all people want to see -anyway right? These are mostly taken from running files in the weave/example -directory and also from the test scripts. Without more information -about what the test actually do, their value is limited. Still, their -here for the curious. Look at the example scripts for more specifics -about what problem was actually solved by each run. These examples are -run under windows 2000 using Microsoft Visual C++ and python2.1 on a -850 MHz PIII laptop with 320 MB of RAM. -Speed up is the improvement (degredation) factor of weave -compared to conventional Python functions. The blitz() -comparisons are shown -compared to NumPy. -

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

inline and ext_tools

-
-

Algorithm

- 		-

Speed up

-
binary search   1.50
fibonacci (recursive)  82.10
fibonacci (loop)   9.17
return None   0.14
map   1.20
dictionary sort   2.54
vector quantization  37.40
-

blitz -- double precision

-
-

Algorithm

- 		-

Speed up

-
a = b + c 512x512   3.05
a = b + c + d 512x512   4.59
5 pt avg. filter, 2D Image 512x512   9.01
Electromagnetics (FDTD) 100x100x100   8.61
-
-

-The benchmarks shown blitz in the best possible light. -NumPy (at least on my machine) is significantly worse for double -precision than it is for single precision calculations. If your -interested in single precision results, you can pretty much divide the -double precision speed up by 3 and you'll -be close. -

-

Inline

-

-inline() compiles and executes C/C++ code on the fly. -Variables in the local and global Python scope are also available in -the C/C++ code. Values are passed to the C/C++ code by assignment much -like variables are passed into a standard Python function. Values are -returned from the C/C++ code through a special argument called -return_val. Also, the contents of mutable objects can be changed within -the C/C++ code and the changes remain after the C code exits and -returns to Python. (more on this later) -

-

Here's a trivial printf example using inline(): -

-
- 

-    >>> import weave    
-    >>> a  = 1
-    >>> weave.inline('printf("%d\\n",a);',['a'])
-    1
-
-
-

In this, its most basic form, inline(c_code, var_list) -requires two arguments. c_code is a string of valid C/C++ -code. var_list is a list of variable names that are -passed from Python into C/C++. Here we have a simple printf -statement that writes the Python variable a to the -screen. The first time you run this, there will be a pause while the -code is written to a .cpp file, compiled into an extension module, -loaded into Python, cataloged for future use, and executed. On windows -(850 MHz PIII), this takes about 1.5 seconds when using Microsoft's C++ -compiler (MSVC) and 6-12 seconds using gcc (mingw32 2.95.2). All -subsequent executions of the code will happen very quickly because the -code only needs to be compiled once. If you kill and restart the -interpreter and then execute the same code fragment again, there will -be a much shorter delay in the fractions of seconds range. This is -because weave stores a catalog of all previously compiled -functions in an on disk cache. When it sees a string that has been -compiled, it loads the already compiled module and executes the -appropriate function.

-

-Note: If you try the printf example in a GUI shell such -as IDLE, PythonWin, PyShell, etc., you're unlikely to see the output. -This is because the C code is writing to stdout, instead of to the GUI -window. This doesn't mean that inline doesn't work in these -environments -- it only means that standard out in C is not the same as -the standard out for Python in these cases. Non input/output functions -will work as expected. -

-

Although effort has been made to reduce the overhead associated with -calling inline, it is still less efficient for simple code snippets -than using equivalent Python code. The simple printf -example is actually slower by 30% or so than using Python print -statement. And, it is not difficult to create code fragments that are -8-10 times slower using inline than equivalent Python. However, for -more complicated algorithms, the speed up can be worth while -- -anywhwere from 1.5- 30 times faster. Algorithms that have to manipulate -Python objects (sorting a list) usually only see a factor of 2 or so -improvement. Algorithms that are highly computational or manipulate -NumPy arrays can see much larger improvements. The examples/vq.py file -shows a factor of 30 or more improvement on the vector quantization -algorithm that is used heavily in information theory and classification -problems. -

-

-

-

More with printf

-

-MSVC users will actually see a bit of compiler output that distutils -does not -supress the first time the code executes:

-
- 
    
    >>> weave.inline(r'printf("%d\n",a);',['a'])
    sc_e013937dbc8c647ac62438874e5795131.cpp
       Creating library C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp
       \Release\sc_e013937dbc8c647ac62438874e5795131.lib and object C:\DOCUME
       ~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_e013937dbc8c64
       7ac62438874e5795131.exp
    1
-
-

-Nothing bad is happening, its just a bit annoying. Anyone know -how to turn this off?

-

This example also demonstrates using 'raw strings'. The r -preceeding the code string in the last example denotes that this is a -'raw string'. In raw strings, the backslash character is not -interpreted as an escape character, and so it isn't necessary to use a -double backslash to indicate that the '\n' is meant to be interpreted -in the C printf statement instead of by Python. If your C -code contains a lot -of strings and control characters, raw strings might make things -easier. -Most of the time, however, standard strings work just as well. -

-

The printf statement in these examples is formatted to -print out integers. What happens if a is a string? inline -will happily, compile a new version of the code to accept strings as -input, -and execute the code. The result?

-
- 
    
    >>> a = 'string'
    >>> weave.inline(r'printf("%d\n",a);',['a'])
    32956972
-
-

In this case, the result is non-sensical, but also non-fatal. In -other situations, it might produce a compile time error because a -is required to be an integer at some point in the code, or it could -produce a segmentation fault. Its possible to protect against passing inline -arguments of the wrong data type by using asserts in Python.

-
- 
    
    >>> a = 'string'
    >>> def protected_printf(a):    
    ...     assert(type(a) == type(1))
    ...     weave.inline(r'printf("%d\n",a);',['a'])
    >>> protected_printf(1)
     1
    >>> protected_printf('string')
    AssertError...
-
-

For printing strings, the format statement needs to be changed. -Also, weave -doesn't convert strings to char*. Instead it uses CXX Py::String type, -so you have to do a little more work. Here we convert it to a C++ -std::string -and then ask cor the char* version.

-
- 
    
    >>> a = 'string'    
    >>> weave.inline(r'printf("%s\n",std::string(a).c_str());',['a'])
    string
-
-

This is a little convoluted. Perhaps strings should convert to -std::string -objects instead of CXX objects. Or maybe to char*. -

-

As in this case, C/C++ code fragments often have to change to accept -different types. For the given printing task, however, C++ streams -provide a way of a single statement that works for integers and -strings. By default, the stream objects live in the std (standard) -namespace and thus require the use of std::.

-
- 
    
    >>> weave.inline('std::cout << a << std::endl;',['a'])
    1    
    >>> a = 'string'
    >>> weave.inline('std::cout << a << std::endl;',['a'])
    string
-
-

Examples using printf and cout are -included in examples/print_example.py. -

-

More examples

-This section shows several more advanced uses of inline. -It includes a few algorithms from the Python Cookbook -that have been re-written in inline C to improve speed as well as a -couple examples using NumPy and wxPython. - -

Binary search

-Lets look at the example of searching a sorted list of integers for a -value. For inspiration, we'll use Kalle Svensson's -binary_search() algorithm from the Python Cookbook. His recipe -follows: -
- 

-    def binary_search(seq, t):
-        min = 0; max = len(seq) - 1
-        while 1:
-            if max < min:
-                return -1
-            m = (min  + max)  / 2
-            if seq[m] < t: 
-                min = m  + 1 
-            elif seq[m] > t: 
-                max = m  - 1 
-            else:
-                return m    
-
-
-This Python version works for arbitrary Python data types. The C -version below is specialized to handle integer values. There is a -little type checking done in Python to assure that we're working with -the correct data types before heading into C. The variables seq -and t don't need to be declared beacuse weave -handles converting and declaring them in the C code. All other -temporary variables such as min, max, etc. must be -declared -- it is C after all. Here's the new mixed Python/C function: -
- 
    
    def c_int_binary_search(seq,t):
        # do a little type checking in Python
        assert(type(t) == type(1))
        assert(type(seq) == type([]))
        
        # now the C code
        code = """
               #line 29 "binary_search.py"
               int val, m, min = 0;  
               int max = seq.length() - 1;
               PyObject *py_val; 
               for(;;)
               {
                   if (max < min  ) 
                   { 
                       return_val =  Py::new_reference_to(Py::Int(-1)); 
                       break;
                   } 
                   m =  (min + max) /2;
                   val =    py_to_int(PyList_GetItem(seq.ptr(),m),"val"); 
                   if (val  < t) 
                       min = m  + 1;
                   else if (val >  t)
                       max = m - 1;
                   else
                   {
                       return_val = Py::new_reference_to(Py::Int(m));
                       break;
                   }
               }
               """
        return inline(code,['seq','t'])
-
-

We have two variables seq and t passed -in. t is guaranteed (by the assert) to be -an integer. Python integers are converted to C int types in the -transition from Python to C. seq is a Python list. By -default, it is translated to a CXX list object. Full documentation for -the CXX library can be found at its website. The basics are that -the CXX provides C++ class equivalents for Python objects that -simplify, or at least object orientify, working with Python objects in -C/C++. For example, seq.length() returns the length of -the list. A little more about -CXX and its class methods, etc. is in the ** type conversions ** -section. -

-

-Note: CXX uses templates and therefore may be a little less portable -than another alternative by Gordan McMillan called SCXX which was -inspired by -CXX. It doesn't use templates so it should compile faster and be more -portable. -SCXX has a few less features, but it appears to me that it would mesh -with -the needs of weave quite well. Hopefully xxx_spec files will be written -for SCXX in the future, and we'll be able to compare on a more -empirical -basis. Both sets of spec files will probably stick around, it just a -question -of which becomes the default. -

-

Most of the algorithm above looks similar in C to the original -Python code. There are two main differences. The first is the setting -of return_val instead of directly returning from the C -code with a return statement. return_val is -an automatically defined variable of type PyObject* that -is returned from the C code back to Python. You'll have to handle -reference counting issues when setting this variable. In this example, -CXX classes and functions handle the dirty work. All CXX functions and -classes live in the namespace Py::. The following code -converts the integer m to a CXX Int() -object and then to a PyObject* with an incremented -reference count using Py::new_reference_to().

-
- 
   
    return_val = Py::new_reference_to(Py::Int(m));
-
-

The second big differences shows up in the retrieval of integer -values from the Python list. The simple Python seq[i] -call balloons into a C Python API call to grab the value out of the -list and then a separate call to py_to_int() that -converts the PyObject* to an integer. py_to_int() -includes both a NULL cheack and a PyInt_Check() call as -well as the conversion call. If either of the checks fail, an exception -is raised. The entire C++ code block is executed with in a try/catch -block that handles exceptions much like Python does. This removes the -need for most error checking code. -

-

It is worth note that CXX lists do have indexing operators that -result in code that looks much like Python. However, the overhead in -using them appears to be relatively high, so the standard Python API -was used on the seq.ptr() which is the underlying PyObject* -of the List object. -

-

The #line directive that is the first line of the C -code block isn't necessary, but it's nice for debugging. If the -compilation fails because of the syntax error in the code, the error -will be reported as an error in the Python file "binary_search.py" with -an offset from the given line number (29 here). -

-

So what was all our effort worth in terms of efficiency? Well not a -lot in this case. The examples/binary_search.py file runs both Python -and C versions of the functions As well as using the standard bisect -module. If we run it on a 1 million element list and run the search -3000 times (for 0- -2999), here are the results we get:

-
- 
   
    C:\home\ej\wrk\scipy\weave\examples> python binary_search.py
    Binary search for 3000 items in 1000000 length list of integers:
     speed in python: 0.159999966621
     speed of bisect: 0.121000051498
     speed up: 1.32
     speed in c: 0.110000014305
     speed up: 1.45
     speed in c(no asserts): 0.0900000333786
     speed up: 1.78
-
-

So, we get roughly a 50-75% improvement depending on whether we use -the Python asserts in our C version. If we move down to searching a -10000 element list, the advantage evaporates. Even smaller lists might -result in the Python version being faster. I'd like to say that moving -to NumPy lists (and getting rid of the GetItem() call) offers a -substantial speed up, but my preliminary efforts didn't produce one. I -think the log(N) algorithm is to blame. Because the algorithm is nice, -there just isn't much time spent computing things, so moving to C isn't -that big of a win. If there are ways to reduce conversion overhead of -values, this may improve the C/Python speed up. Anyone have other -explanations or faster code, please let me know. -

-

Dictionary Sort

-

-The demo in examples/dict_sort.py is another example from the Python -CookBook. This -submission, by Alex Martelli, demonstrates how to return the values -from a dictionary sorted by their keys:

-
- 
       
    def sortedDictValues3(adict):
        keys = adict.keys()
        keys.sort()
        return map(adict.get, keys)
-
-

Alex provides 3 algorithms and this is the 3rd and fastest of the -set. The C version of this same algorithm follows:

-
- 
       
    def c_sort(adict):
        assert(type(adict) == type({}))
        code = """     
        #line 21 "dict_sort.py"  
        Py::List keys = adict.keys();
        Py::List items(keys.length()); keys.sort();     
        PyObject* item = NULL; 
        for(int i = 0;  i < keys.length();i++)
        {
            item = PyList_GET_ITEM(keys.ptr(),i);
            item = PyDict_GetItem(adict.ptr(),item);
            Py_XINCREF(item);
            PyList_SetItem(items.ptr(),i,item);              
        }           
        return_val = Py::new_reference_to(items);
        """   
        return inline_tools.inline(code,['adict'],verbose=1)
-
-

Like the original Python function, the C++ version can handle any -Python dictionary regardless of the key/value pair types. It uses CXX -objects for the most part to declare python types in C++, but uses -Python API calls to manipulate their contents. Again, this choice is -made for speed. The C++ version, while -more complicated, is about a factor of 2 faster than Python.

-
- 
       
    C:\home\ej\wrk\scipy\weave\examples> python dict_sort.py
    Dict sort of 1000 items for 300 iterations:
     speed in python: 0.319999933243
    [0, 1, 2, 3, 4]
     speed in c: 0.151000022888
     speed up: 2.12
    [0, 1, 2, 3, 4]
-
-

-

-

NumPy -- cast/copy/transpose

-CastCopyTranspose is a function called quite heavily by Linear Algebra -routines -in the NumPy library. Its needed in part because of the row-major -memory layout -of multi-demensional Python (and C) arrays vs. the col-major order of -the underlying -Fortran algorithms. For small matrices (say 100x100 or less), a -significant -portion of the common routines such as LU decompisition or singular -value decompostion -are spent in this setup routine. This shouldn't happen. Here is the -Python -version of the function using standard NumPy operations. -
- 
       
    def _castCopyAndTranspose(type, array):
        if a.typecode() == type:
            cast_array = copy.copy(NumPy.transpose(a))
        else:
            cast_array = copy.copy(NumPy.transpose(a).astype(type))
        return cast_array
-
-And the following is a inline C version of the same function: -
- 

-    from weave.blitz_tools import blitz_type_factories
-    from weave import scalar_spec
-    from weave import inline
-    def _cast_copy_transpose(type,a_2d):
-        assert(len(shape(a_2d)) == 2)
-        new_array = zeros(shape(a_2d),type)
-        NumPy_type = scalar_spec.NumPy_to_blitz_type_mapping[type]
-        code = \
-        """  
-        for(int i = 0;i < _Na_2d[0]; i++)  
-            for(int j = 0;  j < _Na_2d[1]; j++)
-                new_array(i,j) = (%s) a_2d(j,i);
-        """ % NumPy_type
-        inline(code,['new_array','a_2d'],
-               type_factories = blitz_type_factories,compiler='gcc')
-        return new_array
-
-
-This example uses blitz++ arrays instead of the standard representation -of NumPy arrays so that indexing is simplier to write. This is -accomplished by passing in the blitz++ "type factories" to override the -standard Python to C++ type conversions. Blitz++ arrays allow you to -write clean, fast code, but they also are sloooow to compile (20 -seconds or more for this snippet). This is why they aren't the default -type used for Numeric arrays (and also because most compilers can't -compile blitz arrays...). inline() is also forced to use -'gcc' as the compiler because the default compiler on Windows (MSVC) -will not compile blitz code. 'gcc' I think will use the standard -compiler on Unix machine instead of explicitly forcing gcc (check this) -Comparisons of the Python vs inline C++ code show a factor of 3 -speed -up. Also shown are the results of an "inplace" transpose routine that -can be used if the output of the linear algebra routine can overwrite -the original matrix (this is often appropriate). This provides another -factor of 2 improvement. -
- 

-     #C:\home\ej\wrk\scipy\weave\examples> python cast_copy_transpose.py
-    # Cast/Copy/Transposing (150,150)array 1 times
-    #  speed in python: 0.870999932289
-    #  speed in c: 0.25
-    #  speed up: 3.48
-    #  inplace transpose c: 0.129999995232
-    #  speed up: 6.70
-
-
-<> - -

wxPython

-inline -knows how to handle wxPython objects. Thats nice in and of -itself, but it also demonstrates that the type conversion mechanism is -reasonably flexible. Chances are, it won't take a ton of effort to -support special types -you might have. The examples/wx_example.py borrows the scrolled window -example from the wxPython demo, accept that it mixes inline C code in -the middle -of the drawing function. -
- 

-    def DoDrawing(self, dc):
        
        red = wxNamedColour("RED");
        blue = wxNamedColour("BLUE");
        grey_brush = wxLIGHT_GREY_BRUSH;
        code = \
        """
        #line 108 "wx_example.py" 
        dc->BeginDrawing();
        dc->SetPen(wxPen(*red,4,wxSOLID));
        dc->DrawRectangle(5,5,50,50);
        dc->SetBrush(*grey_brush);
        dc->SetPen(wxPen(*blue,4,wxSOLID));
        dc->DrawRectangle(15, 15, 50, 50);
        """
        inline(code,['dc','red','blue','grey_brush'])
        
        dc.SetFont(wxFont(14, wxSWISS, wxNORMAL, wxNORMAL))
        dc.SetTextForeground(wxColour(0xFF, 0x20, 0xFF))
        te = dc.GetTextExtent("Hello World")
        dc.DrawText("Hello World", 60, 65)

        dc.SetPen(wxPen(wxNamedColour('VIOLET'), 4))
        dc.DrawLine(5, 65+te[1], 60+te[0], 65+te[1])
        ...
-
-Here, some of the Python calls to wx objects -were just converted to C++ calls. There -isn't any benefit, it just demonstrates the capabilities. You might -want to use this -if you have a computationally intensive loop in your drawing code that -you want to speed up. -On windows, you'll have to use the MSVC compiler if you use the -standard wxPython -DLLs distributed by Robin Dunn. Thats because MSVC and gcc, while -binary -compatible in C, are not binary compatible for C++. In fact, its -probably best, no matter what platform you're on, to specify that inline -use the same -compiler that was used to build wxPython to be on the safe side. There -isn't currently -a way to learn this info from the library -- you just have to know. -Also, at least -on the windows platform, you'll need to install the wxWindows libraries -and link to them. I think there is a way around this, but I haven't -found it yet -- I get some -linking errors dealing with wxString. One final note. You'll probably -have to -tweak weave/wx_spec.py or weave/wx_info.py for your machine's -configuration to -point at the correct directories etc. There. That should sufficiently -scare people -into not even looking at this... :) - -

Keyword Options

-

-The basic definition of the inline() function has a slew -of optional variables. It also takes keyword arguments that are passed -to distutils as compiler options. The following is a -formatted cut/paste of the argument section of inline's -doc-string. It explains all of the variables. Some examples using -various options will follow.

-
- 
       
    def inline(code,arg_names,local_dict = None, global_dict = None, 
               force = 0, 
               compiler='',
               verbose = 0, 
               support_code = None,
               customize=None, 
               type_factories = None, 
               auto_downcast=1,
               **kw):
-
-inline has quite a few options as listed below. Also, the -keyword arguments for distutils extension modules are accepted to -specify extra information needed for compiling. -
-

inline Arguments:

-
-
-
code
-
string. A string of valid C++ code. It should not specify a -return statement. Instead it should assign results that need to be -returned to Python in the return_val.
-
arg_names
-
list of strings. A list of Python variable names that should be -transferred from Python into the C/C++ code.
-
local_dict
-
optional. dictionary. If specified, it is a dictionary of -values that should be used as the local scope for the C/C++ code. If -local_dict is not specified the local dictionary of the calling -function is used.
-
global_dict
-
optional. dictionary. If specified, it is a dictionary of -values that should be used as the global scope for the C/C++ code. If -global_dict is not specified the global dictionary of the calling -function is used.
-
force
-
optional. 0 or 1. default 0. If 1, the C++ code is compiled -every time inline is called. This is really only useful for debugging, -and probably only useful if you're editing support_code a lot.
-
compiler
-
optional. string. The name of compiler to use when compiling. -On windows, it understands 'msvc' and 'gcc' as well as all the compiler -names understood by distutils. On Unix, it'll only understand the -values understoof by distutils. (I should add 'gcc' though to this). -

On windows, the compiler defaults to the Microsoft C++ -compiler. If this isn't available, it looks for mingw32 (the gcc -compiler).

-

On Unix, it'll probably use the same compiler that was used -when compiling Python. Cygwin's behavior should be similar.

-
-
verbose
-
optional. 0,1, or 2. defualt 0. Speficies how much much -information is printed during the compile phase of inlining code. 0 is -silent (except on windows with msvc where it still prints some -garbage). 1 informs you when compiling starts, finishes, and how long -it took. 2 prints out the command lines for the compilation process and -can be useful if you're having problems getting code to work. Its handy -for finding the name of the .cpp file if you need to examine it. -verbose has no affect if the compilation isn't necessary.
-
support_code
-
optional. string. A string of valid C++ code declaring extra -code that might be needed by your compiled function. This could be -declarations of functions, classes, or structures.
-
customize
-
optional. base_info.custom_info object. An alternative way to -specifiy support_code, headers, etc. needed by the function see the -weave.base_info module for more details. (not sure this'll be used -much).
-
type_factories
-
optional. list of type specification factories. These guys are -what convert Python data types to C/C++ data types. If you'd like to -use a different set of type conversions than the default, specify them -here. Look in the type conversions section of the main documentation -for examples.
-
auto_downcast
-
optional. 0 or 1. default 1. This only affects functions that -have Numeric arrays as input variables. Setting this to 1 will cause -all floating point values to be cast as float instead of double if all -the NumPy arrays are of type float. If even one of the arrays has type -double or double complex, all variables maintain there standard types.
-
-
-

Distutils keywords:

-
inline() also accepts a number of distutils -keywords for controlling how the code is compiled. The following -descriptions have been copied from Greg Ward's distutils.extension.Extension -class doc- -strings for convenience: -
-
sources
-
[string] list of source filenames, relative to the distribution -root (where the setup script lives), in Unix form (slash-separated) for -portability. Source files may be C, C++, SWIG (.i), platform-specific -resource files, or whatever else is recognized by the "build_ext" -command as source for a Python extension. Note: The module_path file is -always appended to the front of this list
-
include_dirs
-
[string] list of directories to search for C/C++ header files -(in Unix form for portability)
-
define_macros
-
[(name : string, value : string|None)] list of macros to -define; each macro is defined using a 2-tuple, where 'value' is either -the string to define it to or None to define it without a particular -value (equivalent of "#define FOO" in source or -DFOO on Unix C -compiler command line)
-
undef_macros
-
[string] list of macros to undefine explicitly
-
library_dirs
-
[string] list of directories to search for C/C++ libraries at -link time
-
libraries
-
[string] list of library names (not filenames or paths) to -link against
-
runtime_library_dirs
-
[string] list of directories to search for C/C++ libraries at -run time -(for shared extensions, this is when the extension is loaded)
-
extra_objects
-
[string] list of extra files to link with (eg. object files not -implied by 'sources', static library that must be explicitly specified, -binary resource files, etc.)
-
extra_compile_args
-
[string] any extra platform- and compiler-specific information -to use when compiling the source files in 'sources'. For platforms and -compilers where "command line" makes sense, this is typically a list of -command-line arguments, but for other platforms it could be anything.
-
extra_link_args
-
[string] any extra platform- and compiler-specific information -to use when linking object files together to create the extension (or -to create a new static Python interpreter). Similar interpretation as -for 'extra_compile_args'.
-
export_symbols
-
[string] list of symbols to be exported from a shared -extension. Not used on all platforms, and not generally necessary for -Python extensions, which typically export exactly one symbol: "init" + -extension_name.
-
-
- -

Keyword Option Examples

-We'll walk through several examples here to demonstrate the behavior of -inline and also how the various arguments are used. -In the simplest (most) cases, code and arg_names -are the only arguments that need to be specified. Here's a simple -example -run on Windows machine that has Microsoft VC++ installed. -
- 

-    >>> from weave import inline
-    >>> a = 'string'
-    >>> code = """
-    ...        int l = a.length();
-    ...        return_val = Py::new_reference_to(Py::Int(l));
-    ...        """
-    >>> inline(code,['a'])
-     sc_86e98826b65b047ffd2cd5f479c627f12.cpp
-    Creating
-       library C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ffd2cd5f479c627f12.lib
-    and object C:\DOCUME~ 1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ff
-    d2cd5f479c627f12.exp
-    6
-    >>> inline(code,['a'])
-    6
-
-
-When inline is first run, you'll notice that pause and -some trash printed to the screen. The "trash" is acutually part of the -compilers -output that distutils does not supress. The name of the extension file, -sc_bighonkingnumber.cpp, is generated from the md5 check -sum -of the C/C++ code fragment. On Unix or windows machines with only -gcc installed, the trash will not appear. On the second call, the code -fragment is not compiled since it already exists, and only the answer -is returned. Now kill the interpreter and restart, and run the same -code with -a different string. -
- 

-    >>> from weave import inline
-    >>> a = 'a longer string' 
-    >>> code = """ 
-    ...        int l = a.length();
-    ...        return_val = Py::new_reference_to(Py::Int(l));  
-    ...        """
-    >>> inline(code,['a'])
-    15
-
-
-

-Notice this time, inline() did not recompile the code -because it -found the compiled function in the persistent catalog of functions. -There is -a short pause as it looks up and loads the function, but it is much -shorter than compiling would require. -

-

You can specify the local and global dictionaries if you'd like -(much like exec or eval() in Python), but -if they aren't specified, the "expected" ones are used -- i.e. the ones -from the function that called inline() . This is -accomplished through a little call frame trickery. Here is an example -where the local_dict is specified using -the same code example from above:

-
- 

-    >>> a = 'a longer string'
-    >>> b = 'an even  longer string' 
-    >>> my_dict = {'a':b}
-    >>> inline(code,['a'])
-    15
-    >>> inline(code,['a'],my_dict)
-    21
-
-
-

Everytime, the code is changed, inline -does a recompile. However, changing any of the other options in inline -does not -force a recompile. The force option was added so that one -could force a recompile when tinkering with other variables. In -practice, -it is just as easy to change the code by a single -character -(like adding a space some place) to force the recompile. Note: It -also might be nice to add some methods for purging the cache and on -disk catalogs. -

-

I use verbose sometimes for debugging. When set to 2, -it'll output all the information (including the name of the .cpp file) -that you'd -expect from running a make file. This is nice if you need to examine -the -generated code to see where things are going haywire. Note that error -messages from failed compiles are printed to the screen even if verbose - is set to 0. -

-

The following example demonstrates using gcc instead of the standard -msvc compiler on windows using same code fragment as above. Because the -example has already been compiled, the force=1 flag is -needed to make inline() ignore the previously compiled -version and recompile using gcc. The verbose flag is added to show what -is printed out:

-
- 

-    >>>inline(code,['a'],compiler='gcc',verbose=2,force=1)
-    running build_ext    
-    building 'sc_86e98826b65b047ffd2cd5f479c627f13' extension 
-    c:\gcc-2.95.2\bin\g++.exe -mno-cygwin -mdll -O2 -w -Wstrict-prototypes -IC:
-    \home\ej\wrk\scipy\weave -IC:\Python21\Include -c C:\DOCUME~1\eric\LOCAL
-    S~1\Temp\python21_compiled\sc_86e98826b65b047ffd2cd5f479c627f13.cpp -o C:\D
-    OCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b04
-    7ffd2cd5f479c627f13.o    
-    skipping C:\home\ej\wrk\scipy\weave\CXX\cxxextensions.c (C:\DOCUME~1\eri
-    c\LOCALS~1\Temp\python21_compiled\temp\Release\cxxextensions.o up-to-date)
-    skipping C:\home\ej\wrk\scipy\weave\CXX\cxxsupport.cxx (C:\DOCUME~1\eric
-    \LOCALS~1\Temp\python21_compiled\temp\Release\cxxsupport.o up-to-date)
-    skipping C:\home\ej\wrk\scipy\weave\CXX\IndirectPythonInterface.cxx (C:\
-    DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\indirectpythonin
-    terface.o up-to-date)
-    skipping C:\home\ej\wrk\scipy\weave\CXX\cxx_extensions.cxx (C:\DOCUME~1\
-    eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxx_extensions.o up-to-da
-    te)
-    writing C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86
-    e98826b65b047ffd2cd5f479c627f13.def
-    c:\gcc-2.95.2\bin\dllwrap.exe --driver-name g++ -mno-cygwin -mdll -static -
-    -output-lib C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\l
-    ibsc_86e98826b65b047ffd2cd5f479c627f13.a --def C:\DOCUME~1\eric\LOCALS~1\Te
-    mp\python21_compiled\temp\Release\sc_86e98826b65b047ffd2cd5f479c627f13.def 
-    -s C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e9882
-    6b65b047ffd2cd5f479c627f13.o C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compil
-    ed\temp\Release\cxxextensions.o C:\DOCUME~1\eric\LOCALS~1\Temp\python21_com
-    piled\temp\Release\cxxsupport.o C:\DOCUME~1\eric\LOCALS~1\Temp\python21_com
-    piled\temp\Release\indirectpythoninterface.o C:\DOCUME~1\eric\LOCALS~1\Temp
-    \python21_compiled\temp\Release\cxx_extensions.o -LC:\Python21\libs -lpytho
-    n21 -o C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\sc_86e98826b65b047f
-    fd2cd5f479c627f13.pyd
-    15
-
-
-That's quite a bit of output. verbose=1 just prints the -compile -time. -
- 

-    >>>inline(code,['a'],compiler='gcc',verbose=1,force=1)
-    Compiling code...
-    finished compiling (sec):  6.00800001621
-    15
-
-
-

- Note: I've only used the compiler option for -switching between 'msvc' -and 'gcc' on windows. It may have use on Unix also, but I don't know -yet. -

-

The support_code argument is likely to be used a lot. -It allows you to specify extra code fragments such as function, -structure or class definitions that you want to use in the code -string. Note that changes to support_code do not -force a recompile. The catalog only relies on code (for -performance reasons) to determine whether recompiling is necessary. So, -if you make a change to support_code, you'll need to alter code -in some way or use the force argument to get the code to -recompile. I usually just add some inocuous whitespace to the end of -one of the lines in code somewhere. Here's an example of -defining a separate method for calculating -the string length:

-
- 

-    >>> from weave import inline
-    >>> a = 'a longer string'
-    >>> support_code = """
-    ...                PyObject* length(Py::String a)
-    ...                {
-    ...                    int l = a.length();  
-    ...                    return Py::new_reference_to(Py::Int(l)); 
-    ...                }
-    ...                """        
-    >>> inline("return_val = length(a);",['a'],
-    ...        support_code = support_code)
-    15
-
-
-

-customize is a left over from a previous way of specifying -compiler options. It is a custom_info object that can -specify quite a bit of information about how a file is compiled. These info -objects are the standard way of defining compile information for type -conversion classes. However, I don't think they are as handy here, -especially since we've exposed all the keyword arguments that distutils -can handle. Between these keywords, and the support_code -option, I think customize may be obsolete. We'll see if -anyone cares to use it. If not, it'll get axed in the next version. -

-

The type_factories variable is important to people who -want to -customize the way arguments are converted from Python to C. We'll talk -about -this in the next chapter **xx** of this document when we discuss type -conversions. -

-

auto_downcast handles one of the big type conversion -issues that -is common when using NumPy arrays in conjunction with Python scalar -values. -If you have an array of single precision values and multiply that array -by a Python scalar, the result is upcast to a double precision array -because the -scalar value is double precision. This is not usually the desired -behavior -because it can double your memory usage. auto_downcast -goes -some distance towards changing the casting precedence of arrays and -scalars. -If your only using single precision arrays, it will automatically -downcast all -scalar values from double to single precision when they are passed into -the -C++ code. This is the default behavior. If you want all values to keep -there -default type, set auto_downcast to 0. -

-

-

-

Returning Values

-Python variables in the local and global scope transfer seemlessly from -Python into the C++ snippets. And, if inline were to -completely live up -to its name, any modifications to variables in the C++ code would be -reflected -in the Python variables when control was passed back to Python. For -example, -the desired behavior would be something like: -
- 

-    # THIS DOES NOT WORK
-    >>> a = 1
-    >>> weave.inline("a++;",['a'])
-    >>> a
-    2
-
-
-Instead you get: -
- 

-    >>> a = 1
-    >>> weave.inline("a++;",['a'])
-    >>> a
-    1
-
-
-Variables are passed into C++ as if you are calling a Python function. -Python's calling convention is sometimes called "pass by assignment". -This means its as if a c_a = a assignment is made right -before inline call is made and the c_a -variable is used within the C++ code. Thus, any changes made to c_a -are not reflected in Python's a variable. Things do get a -little more confusing, however, when looking at variables with mutable -types. Changes made in C++ to the contents of mutable types are -reflected in the Python variables. -
- 

-    >>> a= [1,2]
-    >>> weave.inline("PyList_SetItem(a.ptr(),0,PyInt_FromLong(3));",['a'])
-    >>> print a
-    [3, 2]
-
-
-So modifications to the contents of mutable types in C++ are seen when -control -is returned to Python. Modifications to immutable types such as tuples, -strings, and numbers do not alter the Python variables. -If you need to make changes to an immutable variable, you'll need to -assign -the new value to the "magic" variable return_val in C++. -This -value is returned by the inline() function: -
- 

-    >>> a = 1
-    >>> a = weave.inline("return_val = Py::new_reference_to(Py::Int(a+1));",['a'])  
-    >>> a
-    2
-
-
-The return_val variable can also be used to return newly -created values. This is possible by returning a tuple. The following -trivial example illustrates how this can be done: -
- 
       
    # python version
    def multi_return():
        return 1, '2nd'
    
    # C version.
    def c_multi_return():    
        code =  """
     	        py::tuple results(2);
     	        results[0] = 1;
     	        results[1] = "2nd";
     	        return_val = results; 	        
                """
        return inline_tools.inline(code)
-
-

The example is available in examples/tuple_return.py. -It also -has the dubious honor of demonstrating how much inline() -can slow things down. The C version here is about 7-10 times slower -than -the Python -version. Of course, something so trivial has no reason to be written in -C anyway. -

-

The issue with locals()

-

-inline passes the locals() and globals() -dictionaries from Python into the C++ function from the calling -function. It extracts the variables that are used in the C++ code from -these dictionaries, converts then to C++ variables, and then calculates -using them. It seems like it would be trivial, then, after the -calculations were finished to then insert the new values back into the locals() -and globals() dictionaries so that the modified values -were reflected in Python. Unfortunately, as pointed out by the Python -manual, the locals() dictionary is not writable.

-

-I suspect locals() is not writable because there are some -optimizations done to speed lookups of the local namespace. I'm -guessing local lookups don't always look at a dictionary to find -values. Can someone "in the know" confirm or correct this? Another -thing I'd like to know is whether there is a way to write to the local -namespace of another stack frame from C/C++. If so, it would be -possible to have some clean up code in compiled functions that wrote -final values of variables in C++ back to the correct Python stack -frame. I think this goes a long way toward making inline -truely live up to its name. I don't think we'll get to the point of -creating variables in Python for variables created in C -- although I -suppose with a C/C++ parser you could do that also. -

-

-

-

A quick look at the code

-weave generates a C++ file holding an extension function -for each inline code snippet. These file names are -generated using from the md5 signature of the code snippet and saved to -a location specified by the PYTHONCOMPILED environment variable -(discussed later). The cpp files are generally about 200-400 lines long -and include quite a few functions to support type conversions, etc. -However, the actual compiled function is pretty simple. Below is the -familiar printf example: -
- 

-    >>> import weave    
-    >>> a = 1
-    >>> weave.inline('printf("%d\\n",a);',['a'])
-    1
-
-
-And here is the extension function generated by inline: -
- 

-static PyObject* compiled_func(PyObject*self, PyObject* args)
{
    py::object return_val;
    int exception_occured = 0;
    PyObject *py__locals = NULL;
    PyObject *py__globals = NULL;
    PyObject *py_a;
    py_a = NULL;

    if(!PyArg_ParseTuple(args,"OO:compiled_func",&py__locals,&py__globals))
        return NULL;
    try
    {
        PyObject* raw_locals = py_to_raw_dict(py__locals,"_locals");
        PyObject* raw_globals = py_to_raw_dict(py__globals,"_globals");
        /* argument conversion code */
        py_a = get_variable("a",raw_locals,raw_globals);
        int a = convert_to_int(py_a,"a");
        /* inline code */
        /* NDARRAY API VERSION 90907 */
        printf("%d\n",a);    /*I would like to fill in changed    locals and globals here...*/

    }
    catch(...)
    {
        return_val =  py::object();
        exception_occured = 1;
    }
    /* cleanup code */
    if(!(PyObject*)return_val && !exception_occured)
    {

        return_val = Py_None;
    }

    return return_val.disown();
}
-
-Every inline function takes exactly two arguments -- the local and -global -dictionaries for the current scope. All variable values are looked up -out -of these dictionaries. The lookups, along with all inline -code execution, are done within a C++ try block. If the -variables -aren't found, or there is an error converting a Python variable to the -appropriate type in C++, an exception is raised. The C++ exception -is automatically converted to a Python exception by SCXX and returned -to -Python. -The py_to_int() function illustrates how the conversions -and -exception handling works. py_to_int first checks that the given -PyObject* -pointer is not NULL and is a Python integer. If all is well, it calls -the -Python API to convert the value to an int. Otherwise, it -calls -handle_bad_type() which gathers information about what -went wrong -and then raises a SCXX TypeError which returns to Python as a -TypeError. -
- 

-    int py_to_int(PyObject* py_obj,char* name)
-    {
-        if (!py_obj || !PyInt_Check(py_obj))
-            handle_bad_type(py_obj,"int", name);
-        return (int) PyInt_AsLong(py_obj);
-    }
-
-
-
- 

-    void handle_bad_type(PyObject* py_obj, char* good_type, char*  var_name)
-    {
-        char msg[500];
-        sprintf(msg,"received '%s' type instead of '%s' for variable '%s'",
-                find_type(py_obj),good_type,var_name);
-        throw Py::TypeError(msg);
-    }
-    
-    char* find_type(PyObject* py_obj)
-    {
-        if(py_obj == NULL) return "C NULL value";
-        if(PyCallable_Check(py_obj)) return "callable";
-        if(PyString_Check(py_obj)) return "string";
-        if(PyInt_Check(py_obj)) return "int";
-        if(PyFloat_Check(py_obj)) return "float";
-        if(PyDict_Check(py_obj)) return "dict";
-        if(PyList_Check(py_obj)) return "list";
-        if(PyTuple_Check(py_obj)) return "tuple";
-        if(PyFile_Check(py_obj)) return "file";
-        if(PyModule_Check(py_obj)) return "module";
-        
-        //should probably do more interagation (and thinking) on these.
-        if(PyCallable_Check(py_obj) && PyInstance_Check(py_obj)) return "callable";
-        if(PyInstance_Check(py_obj)) return "instance"; 
-        if(PyCallable_Check(py_obj)) return "callable";
-        return "unkown type";
-    }
-
-
-Since the inline is also executed within the try/catch -block, you can use CXX exceptions within your code. It is usually a bad -idea -to directly return from your code, even if an error -occurs. This -skips the clean up section of the extension function. In this simple -example, -there isn't any clean up code, but in more complicated examples, there -may -be some reference counting that needs to be taken care of here on -converted -variables. To avoid this, either uses exceptions or set return_val -to NULL and use if/then's to skip code -after errors. - -

Technical Details

-

-There are several main steps to using C/C++ code withing Python: -

-
    -
  1. Type conversion
    2. -
  2. Generating C/C++ code
    3. -
  3. Compile the code to an extension module
    4. -
  4. Catalog (and cache) the function for future use
    5. -
-

-Items 1 and 2 above are related, but most easily discussed separately. -Type conversions are customizable by the user if needed. Understanding -them is pretty important for anything beyond trivial uses of inline. -Generating the C/C++ code is handled by ext_function and ext_module -classes and . For the most part, compiling the code is handled by -distutils. Some customizations were needed, but they were relatively -minor and do not require changes to distutils itself. Cataloging is -pretty simple in concept, but surprisingly required the most code to -implement (and still likely needs some work). So, this section covers -items 1 and 4 from the list. Item 2 is covered later in the chapter -covering the ext_tools module, and distutils is covered -by a completely separate document xxx. -

-

Passing Variables in/out of the C/C++ code

- -Note: Passing variables into the C code is pretty straight forward, but -there are subtlties to how variable modifications in C are returned to -Python. see Returning Values for a -more thorough discussion of this issue. - -

Type Conversions

- -Note: Maybe xxx_converter instead of xxx_specification -is a more descriptive name. Might change in future version? - -

By default, inline() makes the following type -conversions between -Python and C++ types. -

-

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

Default Data Type Conversions

-
-

Python

- 		-

C++

-
   int   int
   float   double
   complex   std::complex
   string   py::string
   list   py::list
   dict   py::dict
   tuple   py::tuple
   file   FILE*
   callable   py::object
   instance   py::object
   numpy.ndarray   PyArrayObject*
   wxXXX   wxXXX*
-
-

-The Py:: namespace is defined by the SCXX library which -has C++ class -equivalents for many Python types. std:: is the namespace -of the -standard library in C++. -

-

-Note: -

    -
  • I haven't figured out how to handle long int yet (I -think they are currenlty converted to int - - check this).
    • -
  • Hopefully VTK will be added to the list soon
    • -
- -

-

Python to C++ conversions fill in code in several locations in the -generated -inline extension function. Below is the basic template for -the -function. This is actually the exact code that is generated by calling -weave.inline("").

-
- 

-
-
-The /* inline code */ section is filled with the code -passed to -the inline() function call. The /*argument -convserion code*/ and /* cleanup code */ -sections are filled with code that handles conversion from Python to -C++ -types and code that deallocates memory or manipulates reference counts -before -the function returns. The following sections demostrate how these two -areas -are filled in by the default conversion methods. - Note: I'm not sure I have reference counting correct on a few of -these. The only thing I increase/decrease the ref count on is NumPy -arrays. If you -see an issue, please let me know. - -

NumPy Argument Conversion

-Integer, floating point, and complex arguments are handled in a very -similar -fashion. Consider the following inline function that has a single -integer variable passed in: -
- 

-    >>> a = 1
-    >>> inline("",['a'])
-
-
-The argument conversion code inserted for a is: -
- 

-    /* argument conversion code */
-    int a = py_to_int (get_variable("a",raw_locals,raw_globals),"a");
-
-
-get_variable() reads the variable a -from the local and global namespaces. py_to_int() has the -following -form: -
- 

-    static int py_to_int(PyObject* py_obj,char* name)
-    {
-        if (!py_obj || !PyInt_Check(py_obj))
-            handle_bad_type(py_obj,"int", name);
-        return (int) PyInt_AsLong(py_obj);
-    }
-
-
-Similarly, the float and complex conversion routines look like: -
- 
    
    static double py_to_float(PyObject* py_obj,char* name)
    {
        if (!py_obj || !PyFloat_Check(py_obj))
            handle_bad_type(py_obj,"float", name);
        return PyFloat_AsDouble(py_obj);
    }
    
    static std::complex py_to_complex(PyObject* py_obj,char* name)
    {
        if (!py_obj || !PyComplex_Check(py_obj))
            handle_bad_type(py_obj,"complex", name);
        return std::complex(PyComplex_RealAsDouble(py_obj),
                                    PyComplex_ImagAsDouble(py_obj));    
    }
-
-NumPy conversions do not require any clean up code. - -

String, List, Tuple, and Dictionary Conversion

-Strings, Lists, Tuples and Dictionary conversions are all converted to -SCXX types by default. -For the following code, -
- 

-    >>> a = [1]
-    >>> inline("",['a'])
-
-
-The argument conversion code inserted for a is: -
- 

-    /* argument conversion code */
-    Py::List a = py_to_list (get_variable("a",raw_locals,raw_globals),"a");
-
-
-get_variable() reads the variable a -from the local and global namespaces. py_to_list() and -its -friends has the following form: -
- 
    
    static Py::List py_to_list(PyObject* py_obj,char* name)
    {
        if (!py_obj || !PyList_Check(py_obj))
            handle_bad_type(py_obj,"list", name);
        return Py::List(py_obj);
    }
    
    static Py::String py_to_string(PyObject* py_obj,char* name)
    {
        if (!PyString_Check(py_obj))
            handle_bad_type(py_obj,"string", name);
        return Py::String(py_obj);
    }

    static Py::Dict py_to_dict(PyObject* py_obj,char* name)
    {
        if (!py_obj || !PyDict_Check(py_obj))
            handle_bad_type(py_obj,"dict", name);
        return Py::Dict(py_obj);
    }
    
    static Py::Tuple py_to_tuple(PyObject* py_obj,char* name)
    {
        if (!py_obj || !PyTuple_Check(py_obj))
            handle_bad_type(py_obj,"tuple", name);
        return Py::Tuple(py_obj);
    }
-
-SCXX handles reference counts on for strings, lists, tuples, and -dictionaries, -so clean up code isn't necessary. - -

File Conversion

-For the following code, -
- 

-    >>> a = open("bob",'w')  
-    >>> inline("",['a'])
-
-
-The argument conversion code is: -
- 

-    /* argument conversion code */
-    PyObject* py_a = get_variable("a",raw_locals,raw_globals);
-    FILE* a = py_to_file(py_a,"a");
-
-
-get_variable() reads the variable a -from the local and global namespaces. py_to_file() -converts -PyObject* to a FILE* and increments the reference count of the -PyObject*: -
- 

-    FILE* py_to_file(PyObject* py_obj, char* name)
-    {
-        if (!py_obj || !PyFile_Check(py_obj))
-            handle_bad_type(py_obj,"file", name);
-    
-        Py_INCREF(py_obj);
-        return PyFile_AsFile(py_obj);
-    }
-
-
-Because the PyObject* was incremented, the clean up code needs to -decrement -the counter -
- 

-    /* cleanup code */
-    Py_XDECREF(py_a);
-
-
-Its important to understand that file conversion only works on actual -files -- -i.e. ones created using the open() command in Python. It -does -not support converting arbitrary objects that support the file -interface into -C FILE* pointers. This can affect many things. For -example, in -initial printf() examples, one might be tempted to solve -the problem of C and Python IDE's (PythonWin, PyCrust, etc.) writing to -different -stdout and stderr by using fprintf() and passing in sys.stdout -and sys.stderr. For example, instead of -
- 

-    >>> weave.inline('printf("hello\\n");')
-
-
-You might try: -
- 

-    >>> buf = sys.stdout
-    >>> weave.inline('fprintf(buf,"hello\\n");',['buf'])
-
-
-This will work as expected from a standard python interpreter, but in -PythonWin, -the following occurs: -
- 

-    >>> buf = sys.stdout
-    >>> weave.inline('fprintf(buf,"hello\\n");',['buf'])
-    Traceback (most recent call last):
-        File "", line 1, in ?
        File "C:\Python21\weave\inline_tools.py", line 315, in inline
            auto_downcast = auto_downcast,
        File "C:\Python21\weave\inline_tools.py", line 386, in compile_function
            type_factories = type_factories)
        File "C:\Python21\weave\ext_tools.py", line 197, in __init__
            auto_downcast, type_factories)
        File "C:\Python21\weave\ext_tools.py", line 390, in assign_variable_types
            raise TypeError, format_error_msg(errors)
        TypeError: {'buf': "Unable to convert variable 'buf' to a C++ type."}
-
-The traceback tells us that inline() was unable to -convert 'buf' to a -C++ type (If instance conversion was implemented, the error would have -occurred at runtime instead). Why is this? Let's look at what the buf -object really is: -
- 

-    >>> buf
-    pywin.framework.interact.InteractiveView instance at 00EAD014
-
-
-PythonWin has reassigned sys.stdout to a special object -that implements the Python file interface. This works great in Python, -but since the special object doesn't have a FILE* pointer underlying -it, fprintf doesn't know what to do with it (well this will be the -problem when instance conversion is implemented...). - -

Callable, Instance, and Module Conversion

-Note: Need to look into how ref counts should be handled. Also, -Instance and Module conversion are not currently implemented. - -
- 

-    >>> def a(): 
-        pass
-    >>> inline("",['a'])
-
-
-Callable and instance variables are converted to PyObject*. Nothing is -done -to there reference counts. -
- 

-    /* argument conversion code */
-    PyObject* a = py_to_callable(get_variable("a",raw_locals,raw_globals),"a");
-
-
-get_variable() reads the variable a -from the local and global namespaces. The py_to_callable() -and -py_to_instance() don't currently increment the ref count. -
- 
    
    PyObject* py_to_callable(PyObject* py_obj, char* name)
    {
        if (!py_obj || !PyCallable_Check(py_obj))
            handle_bad_type(py_obj,"callable", name);    
        return py_obj;
    }

    PyObject* py_to_instance(PyObject* py_obj, char* name)
    {
        if (!py_obj || !PyFile_Check(py_obj))
            handle_bad_type(py_obj,"instance", name);    
        return py_obj;
    }
-
-There is no cleanup code for callables, modules, or instances. - -

Customizing Conversions

-

-Converting from Python to C++ types is handled by xxx_specification -classes. A type specification class actually serve in two related but -different roles. The first is in determining whether a Python variable -that needs to be converted should be represented by the given class. -The second is as a code generator that generate C++ code needed to -convert from Python to C++ types for a specific variable. -

-

When

-
- 

-    >>> a = 1
-    >>> weave.inline('printf("%d",a);',['a'])
-
-
-is called for the first time, the code snippet has to be compiled. In -this process, the variable 'a' is tested against a list of type -specifications (the default list is stored in weave/ext_tools.py). The first -specification in the list is used to represent the variable. -

Examples of xxx_specification are scattered throughout -numerous "xxx_spec.py" files in the weave package. -Closely related to the xxx_specification classes are yyy_info -classes. These classes contain compiler, header, and support code -information necessary for including a certain set of capabilities (such -as blitz++ or CXX support) -in a compiled module. xxx_specification classes have one -or more -yyy_info classes associated with them. -If you'd like to define your own set of type specifications, the -current best route -is to examine some of the existing spec and info files. Maybe looking -over -sequence_spec.py and cxx_info.py are a good place to start. After -defining specification classes, you'll need to pass them into inline -using the type_factories argument. A lot of times you may -just want to change how a specific variable type is represented. Say -you'd rather have Python strings converted to std::string -or maybe char* instead of using the CXX string object, -but would like all other type conversions to have default behavior. -This requires that a new specification class that handles strings -is written and then prepended to a list of the default type -specifications. Since -it is closer to the front of the list, it effectively overrides the -default -string specification. -The following code demonstrates how this is done: -... -

-

The Catalog

-

-catalog.py has a class called catalog that -helps keep track of previously compiled functions. This prevents inline() -and related functions from having to compile functions everytime they -are called. Instead, catalog will check an in memory cache to see if -the function has already been loaded into python. If it hasn't, then it -starts searching through persisent catalogs on disk to see if it finds -an entry for the given function. By saving information about compiled -functions to disk, it isn't -necessary to re-compile functions everytime you stop and restart the -interpreter. -Functions are compiled once and stored for future use. -

-

When inline(cpp_code) is called the following things -happen: -

-
    -
  1. A fast local cache of functions is checked for the last function -called for cpp_code. If an entry for cpp_code -doesn't exist in the cache or the cached function call fails (perhaps -because the function doesn't have compatible types) then the next step -is to check the catalog.
    2. -
  2. The catalog class also keeps an in-memory cache with a list of -all the functions compiled for cpp_code. If cpp_code -has ever been called, then this cache will be present (loaded from -disk). If the cache isn't present, then it is loaded from disk. -

    If the cache is present, each function in the cache is called -until one is found that was compiled for the correct argument types. If -none of the functions work, a new function is compiled with the given -argument types. This function is written to the on-disk catalog as well -as into the in-memory cache.

    -
    3. -
  3. When a lookup for cpp_code fails, the catalog -looks through the on-disk function catalogs for the entries. The -PYTHONCOMPILED variable determines where to search for these catalogs -and in what order. If PYTHONCOMPILED is not present several platform -dependent locations are searched. All functions found for cpp_code -in the path are loaded into the in-memory cache with functions found -earlier in the search path closer to the front of the call list. -

    If the function isn't found in the on-disk catalog, then the -function is compiled, written to the first writable directory in the -PYTHONCOMPILED path, and also loaded into the in-memory cache.

    -
    4. -
- -

Function Storage: How functions are stored in caches and on disk

-

-Function caches are stored as dictionaries where the key is the entire -C++ -code string and the value is either a single function (as in the "level -1" -cache) or a list of functions (as in the main catalog cache). On disk -catalogs are stored in the same manor using standard Python shelves. -

-

Early on, there was a question as to whether md5 check sums of the -C++ -code strings should be used instead of the actual code strings. I think -this -is the route inline Perl took. Some (admittedly quick) tests of the md5 -vs. -the entire string showed that using the entire string was at least a -factor of 3 or 4 faster for Python. I think this is because it is more -time consuming to compute the md5 value than it is to do look-ups of -long -strings in the dictionary. Look at the examples/md5_speed.py file for -the -test run. -

-

Catalog search paths and the PYTHONCOMPILED variable

-

-The default location for catalog files on Unix is is -~/.pythonXX_compiled where XX is version of Python being used. If this -directory doesn't exist, it is created the first time a catalog is -used. The directory must be writable. If, for any reason it isn't, then -the catalog attempts to create a directory based on your user id in the -/tmp directory. The directory permissions are set so that only you have -access to the directory. If this fails, I think you're out of luck. I -don't think either of these should ever fail though. On Windows, a -directory called pythonXX_compiled is created in the user's temporary -directory.

-

The actual catalog file that lives in this directory is a Python -shelve with -a platform specific name such as "nt21compiled_catalog" so that -multiple OSes -can share the same file systems without trampling on each other. Along -with -the catalog file, the .cpp and .so or .pyd files created by inline will -live -in this directory. The catalog file simply contains keys which are the -C++ -code strings with values that are lists of functions. The function -lists point -at functions within these compiled modules. Each function in the lists -executes the same C++ code string, but compiled for different input -variables. -

-

You can use the PYTHONCOMPILED environment variable to specify -alternative -locations for compiled functions. On Unix this is a colon (':') -separated -list of directories. On windows, it is a (';') separated list of -directories. -These directories will be searched prior to the default directory for a -compiled function catalog. Also, the first writable directory in the -list -is where all new compiled function catalogs, .cpp and .so or .pyd files -are -written. Relative directory paths ('.' and '..') should work fine in -the -PYTHONCOMPILED variable as should environement variables. -

-

There is a "special" path variable called MODULE that can be placed -in the PYTHONCOMPILED variable. It specifies that the compiled catalog -should -reside in the same directory as the module that called it. This is -useful -if an admin wants to build a lot of compiled functions during the build -of a package and then install them in site-packages along with the -package. -User's who specify MODULE in their PYTHONCOMPILED variable will have -access -to these compiled functions. Note, however, that if they call the -function -with a set of argument types that it hasn't previously been built for, -the -new function will be stored in their default directory (or some other -writable -directory in the PYTHONCOMPILED path) because the user will not have -write -access to the site-packages directory. -

-

An example of using the PYTHONCOMPILED path on bash follows:

-
- 

-    PYTHONCOMPILED=MODULE:/some/path;export PYTHONCOMPILED;
-
-
-If you are using python21 on linux, and the module bob.py in -site-packages -has a compiled function in it, then the catalog search order when -calling that -function for the first time in a python session would be: -
- 

-    /usr/lib/python21/site-packages/linuxpython_compiled
-    /some/path/linuxpython_compiled
-    ~/.python21_compiled/linuxpython_compiled
-
-
-The default location is always included in the search path. -

Note: hmmm. see a possible problem here. I should probably make -a sub- -directory such as /usr/lib/python21/site- -packages/python21_compiled/linuxpython_compiled so that library files -compiled with python21 are tried to link with python22 files in some -strange scenarios. Need to check this. -

-

The in-module cache (in weave.inline_tools reduces the -overhead of calling inline functions by about a factor of 2. It can be -reduced a little more for type loop calls where the same function is -called over and over again if the cache was a single value instead of a -dictionary, but the benefit is very small (less than 5%) and the -utility is quite a bit less. So, we'll stick with a dictionary as the -cache. -

-

- -

Blitz

- Note: most of this section is lifted from old documentation. It -should be -pretty accurate, but there may be a few discrepancies. -

weave.blitz() compiles NumPy Python expressions for -fast execution. For most applications, compiled expressions should -provide a factor of 2-10 speed-up over NumPy arrays. Using compiled -expressions is meant to be as unobtrusive as possible and works much -like pythons exec statement. As an example, the following code fragment -takes a 5 point average of the 512x512 2d image, b, and stores it in -array, a:

-
- 

-    from scipy import *  # or from NumPy import *
-    a = ones((512,512), Float64) 
-    b = ones((512,512), Float64) 
-    # ...do some stuff to fill in b...
-    # now average
-    a[1:-1,1:-1] =  (b[1:-1,1:-1] + b[2:,1:-1] + b[:-2,1:-1] \
-                   + b[1:-1,2:] + b[1:-1,:-2]) / 5.
-
-
-To compile the expression, convert the expression to a string by -putting -quotes around it and then use weave.blitz: -
- 

-    import weave
-    expr = "a[1:-1,1:-1] =  (b[1:-1,1:-1] + b[2:,1:-1] + b[:-2,1:-1]" \
-                          "+ b[1:-1,2:] + b[1:-1,:-2]) / 5."
-    weave.blitz(expr)
-
-
-The first time weave.blitz is run for a given expression -and set of arguements, C++ code that accomplishes the exact same task -as the Python expression is generated and compiled to an extension -module. This can take up to a couple of minutes depending on the -complexity of the function. Subsequent calls to the function are very -fast. Futher, the generated module is saved between program executions -so that the compilation is only done once for a given expression and -associated set of array types. If the given expression -is executed with a new set of array types, the code most be compiled -again. This -does not overwrite the previously compiled function -- both of them are -saved and -available for exectution. -

The following table compares the run times for standard NumPy code -and compiled code for the 5 point averaging. -

-

-
- - - - - - - - - - - - - - - - - - - -
MethodRun Time (seconds)
Standard NumPy0.46349
blitz (1st time compiling) 78.95526
blitz (subsequent calls)0.05843 (factor of 8 speedup)
-
-

-These numbers are for a 512x512 double precision image run on a 400 MHz -Celeron processor under RedHat Linux 6.2. -

-

Because of the slow compile times, its probably most effective to -develop algorithms as you usually do using the capabilities of scipy or -the NumPy module. Once the algorithm is perfected, put quotes around it -and execute it using weave.blitz. This provides the -standard rapid prototyping strengths of Python and results in -algorithms that run close to that of hand coded C or Fortran. -

-

Requirements

-Currently, the weave.blitz has only been tested under -Linux with gcc-2.95-3 and on Windows with Mingw32 (2.95.2). Its -compiler requirements are pretty heavy duty (see the blitz++ home page), so it -won't work with just any compiler. Particularly MSVC++ isn't up to -snuff. A number of other compilers such as KAI++ will also work, but my -suspicions are that gcc will get the most use. - -

Limitations

-
    -
  1. Currently, weave.blitz handles all standard -mathematic -operators except for the ** power operator. The built-in -trigonmetric, log, floor/ceil, and fabs functions might work (but -haven't been tested). It also handles all types of array indexing -supported by the NumPy module. numarray's NumPy compatible array -indexing modes are likewise supported, but numarray's enhanced -(array based) indexing modes are not supported. -

    weave.blitz does not currently support operations -that use array broadcasting, nor have any of the special purpose -functions in NumPy such as take, compress, etc. been implemented. Note -that there are no obvious reasons why most of this functionality cannot -be added to scipy.weave, so it will likely trickle into future -versions. Using slice() objects directly instead of start:stop:step -is also not supported.

    -
    2. -
  2. Currently Python only works on expressions that include -assignment such as -
    - 
    
-    >>> result = b + c + d
-
    -
    -This means that the result array must exist before calling weave.blitz. -Future versions will allow the following: -
    - 
    
-    >>> result = weave.blitz_eval("b + c + d")
-
    -
    -
    3. -
  3. weave.blitz works best when algorithms can be -expressed in a "vectorized" form. Algorithms that have a large number -of if/thens and other conditions are better hand written in C or -Fortran. Further, the restrictions imposed by requiring vectorized -expressions sometimes preclude the use of more efficient data -structures or algorithms. For maximum speed in these cases, hand-coded -C or Fortran code is the only way to go.
    4. -
  4. weave.blitz can produce different results than -NumPy -in certain situations. It can happen when the array receiving the -results of a calculation is also used during the calculation. The NumPy -behavior is to carry out the entire calculation on the right hand side -of an equation and store it in a temporary array. This temprorary array -is assigned to the array on the left hand side of the equation. blitz, -on the other hand, does a "running" calculation of the array elements -assigning values from the right hand -side to the elements on the left hand side immediately after they are -calculated. -Here is an example, provided by Prabhu Ramachandran, where this -happens: -
    - 
    
-        # 4 point average.
-        >>> expr = "u[1:-1, 1:-1] = (u[0:-2, 1:-1] + u[2:, 1:-1] + "\
-        ...                "u[1:-1,0:-2] + u[1:-1, 2:])*0.25"
-        >>> u = zeros((5, 5), 'd'); u[0,:] = 100
-        >>> exec (expr)
-        >>> u
-        array([[ 100.,  100.,  100.,  100.,  100.],
-               [   0.,   25.,   25.,   25.,    0.],
-               [   0.,    0.,    0.,    0.,    0.],
-               [   0.,    0.,    0.,    0.,    0.],
-               [   0.,    0.,    0.,    0.,    0.]])
-        
-        >>> u = zeros((5, 5), 'd'); u[0,:] = 100
-        >>> weave.blitz (expr)
-        >>> u
-        array([[ 100.  ,  100.       ,  100.       ,  100.       ,  100. ],
-               [   0.  ,   25.       ,   31.25     ,   32.8125   ,    0. ],
-               [   0.  ,    6.25     ,    9.375    ,   10.546875 ,    0. ],
-               [   0.  ,    1.5625   ,    2.734375 ,    3.3203125,    0. ],
-               [   0.  ,    0.       ,    0.       ,    0.       ,    0. ]])    
-
    -
    -You can prevent this behavior by using a temporary array. -
    - 
    
-        >>> u = zeros((5, 5), 'd'); u[0,:] = 100
-        >>> temp = zeros((4, 4), 'd');
-        >>> expr = "temp = (u[0:-2, 1:-1] + u[2:, 1:-1] + "\
-        ...        "u[1:-1,0:-2] + u[1:-1, 2:])*0.25;"\
-        ...        "u[1:-1,1:-1] = temp"
-        >>> weave.blitz (expr)
-        >>> u
-        array([[ 100.,  100.,  100.,  100.,  100.],
-               [   0.,   25.,   25.,   25.,    0.],
-               [   0.,    0.,    0.,    0.,    0.],
-               [   0.,    0.,    0.,    0.,    0.],
-               [   0.,    0.,    0.,    0.,    0.]])
-
    -
    -
    5. -
  5. One other point deserves mention lest people be confused. weave.blitz -is not a general purpose Python->C compiler. It only works for -expressions that contain NumPy arrays and/or Python scalar values. This -focused scope concentrates effort on the compuationally intensive -regions of the program and sidesteps the difficult issues associated -with a general purpose Python->C compiler.
    6. -
- -

NumPy efficiency issues: What compilation buys you

-Some might wonder why compiling NumPy expressions to C++ is beneficial -since operations on NumPy array operations are already executed within -C loops. The problem is that anything other than the simplest -expression are executed in less than optimal fashion. Consider the -following NumPy expression: -
- 

-    a = 1.2 * b + c * d
-
-
-When NumPy calculates the value for the 2d array, a, it -does the following steps: -
- 

-    temp1 = 1.2 * b
-    temp2 = c * d
-    a = temp1 + temp2
-
-
-Two things to note. Since c is an (perhaps large) array, -a large temporary array must be created to store the results of 1.2 -* b. The same is true for temp2. Allocation is -slow. The second thing is that we have 3 loops executing, one to -calculate temp1, one for temp2 and one for -adding them up. A C loop for the same problem might look like: -
- 

-    for(int i = 0; i < M; i++)
-        for(int j = 0; j < N; j++)
-            a[i,j] = 1.2 * b[i,j] + c[i,j] * d[i,j]
-
-
-Here, the 3 loops have been fused into a single loop and there is no -longer -a need for a temporary array. This provides a significant speed -improvement -over the above example (write me and tell me what you get). -

So, converting NumPy expressions into C/C++ loops that fuse the -loops and eliminate temporary arrays can provide big gains. The goal -then,is to convert NumPy expression to C/C++ loops, compile them in an -extension module, and then call the compiled extension function. The -good news is that there is an obvious correspondence between the NumPy -expression above and the C loop. The bad news is that NumPy is -generally much more powerful than this simple example illustrates and -handling all possible indexing possibilities results in loops that are -less than straight forward to write. (take a peak in NumPy for -confirmation). Luckily, there are several available tools that simplify -the process. -

-

The Tools

-weave.blitz relies heavily on several remarkable tools. On -the Python side, the main facilitators are Jermey Hylton's parser -module and Travis Oliphant's NumPy module. On the compiled language -side, -Todd Veldhuizen's blitz++ array library, written in C++ (shhhh. don't -tell David Beazley), does the heavy lifting. Don't assume that, because -it's C++, it's much slower than C or Fortran. Blitz++ uses a jaw -dropping array of template techniques (metaprogramming, template -expression, etc) to convert innocent looking and readable C++ -expressions into to code that usually executes within a few percentage -points of Fortran code for the same problem. This is good. -Unfortunately all the template raz-ma-taz is very expensive to compile, -so the 200 line extension modules often take 2 or more minutes to -compile. This isn't so good. weave.blitz works to -minimize this issue by remembering where compiled modules live and -reusing them instead of re-compiling every time a program is re-run. - -

Parser

-Tearing NumPy expressions apart, examining the pieces, and then -rebuilding them as C++ (blitz) expressions requires a parser of some -sort. I can imagine someone attacking this problem with regular -expressions, but it'd likely be ugly and fragile. Amazingly, Python -solves this problem for us. It actually exposes its parsing engine to -the world through the parser module. The following -fragment creates an Abstract Syntax Tree (AST) object for the -expression and then converts to a (rather unpleasant looking) deeply -nested list representation of the tree. -
- 

-    >>> import parser
-    >>> import scipy.weave.misc
-    >>> ast = parser.suite("a = b * c + d")
-    >>> ast_list = ast.tolist()
-    >>> sym_list = scipy.weave.misc.translate_symbols(ast_list)
-    >>> pprint.pprint(sym_list)
-    ['file_input',
-     ['stmt',
-      ['simple_stmt',
-       ['small_stmt',
-        ['expr_stmt',
-         ['testlist',
-          ['test',
-           ['and_test',
-            ['not_test',
-             ['comparison',
-              ['expr',
-               ['xor_expr',
-                ['and_expr',
-                 ['shift_expr',
-                  ['arith_expr',
-                   ['term',
-                    ['factor', ['power', ['atom', ['NAME', 'a']]]]]]]]]]]]]]],
-         ['EQUAL', '='],
-         ['testlist',
-          ['test',
-           ['and_test',
-            ['not_test',
-             ['comparison',
-              ['expr',
-               ['xor_expr',
-                ['and_expr',
-                 ['shift_expr',
-                  ['arith_expr',
-                   ['term',
-                    ['factor', ['power', ['atom', ['NAME', 'b']]]],
-                    ['STAR', '*'],
-                    ['factor', ['power', ['atom', ['NAME', 'c']]]]],
-                   ['PLUS', '+'],
-                   ['term',
-                    ['factor', ['power', ['atom', ['NAME', 'd']]]]]]]]]]]]]]]]],
-       ['NEWLINE', '']]],
-     ['ENDMARKER', '']]
-
-
-Despite its looks, with some tools developed by Jermey H., its possible -to search these trees for specific patterns (sub-trees), extract the -sub-tree, manipulate them converting python specific code fragments -to blitz code fragments, and then re-insert it in the parse tree. The -parser -module documentation has some details on how to do this. Traversing the -new blitzified tree, writing out the terminal symbols as you go, -creates -our new blitz++ expression string. - -

Blitz and NumPy

-The other nice discovery in the project is that the data structure used -for NumPy arrays and blitz arrays is nearly identical. NumPy stores -"strides" as byte offsets and blitz stores them as element offsets, but -other than that, they are the same. Further, most of the concept and -capabilities of the two libraries are remarkably similar. It is -satisfying that two completely different implementations solved the -problem with similar basic architectures. It is also fortuitous. The -work involved in converting NumPy expressions to blitz expressions was -greatly diminished. -As an example, consider the code for slicing an array in Python with a -stride: -
- 

-    >>> a = b[0:4:2] + c
-    >>> a
-    [0,2,4]
-
-
-In Blitz it is as follows: -
- 

-    Array<2,int> b(10);
-    Array<2,int> c(3);
-    // ...
-    Array<2,int> a = b(Range(0,3,2)) + c;
-
-
-Here the range object works exactly like Python slice objects with the -exception -that the top index (3) is inclusive where as Python's (4) is exclusive. -Other differences include the type declaraions in C++ and parentheses -instead of brackets for indexing arrays. Currently, weave.blitz -handles the inclusive/exclusive issue by subtracting one from upper -indices during the -translation. An alternative that is likely more robust/maintainable in -the long run, is to write a PyRange class that behaves like Python's -range. This is likely very easy. -

The stock blitz also doesn't handle negative indices in ranges. The -current implementation of the blitz() has a partial -solution to this problem. It calculates and index that starts with a -'-' sign by subtracting it from the maximum index in the array so that: -

-
- 

-                    upper index limit
-                        /-----\
-    b[:-1] -> b(Range(0,Nb[0]-1-1))
-
-
-This approach fails, however, when the top index is calculated from -other values. In the following scenario, if i+j evaluates -to a negative value, the compiled code will produce incorrect results -and could even core- -dump. Right now, all calculated indices are assumed to be positive. -
- 

-    b[:i-j] -> b(Range(0,i+j))
-
-
-A solution is to calculate all indices up front using if/then to handle -the -+/- cases. This is a little work and results in more code, so it hasn't -been -done. I'm holding out to see if blitz++ can be modified to handle -negative -indexing, but haven't looked into how much effort is involved yet. -While it needs fixin', I don't think there is a ton of code where this -is an issue. -

The actual translation of the Python expressions to blitz -expressions is currently a two part process. First, all x:y:z slicing -expression are removed -from the AST, converted to slice(x,y,z) and re-inserted into the tree. -Any -math needed on these expressions (subtracting from the maximum index, -etc.) are also preformed here. _beg and _end are used as special -variables that are defined as blitz::fromBegin and blitz::toEnd.

-
- 

-    a[i+j:i+j+1,:] = b[2:3,:] 
-
-
-becomes a more verbose: -
- 

-    a[slice(i+j,i+j+1),slice(_beg,_end)] = b[slice(2,3),slice(_beg,_end)]
-
-
-The second part does a simple string search/replace to convert to a -blitz expression with the following translations: -
- 

-    slice(_beg,_end) -> _all  # not strictly needed, but cuts down on code.
-    slice            -> blitz::Range
-    [                -> (
-    ]                -> )
-    _stp             -> 1
-
-
-_all is defined in the compiled function as blitz::Range.all(). -These translations could of course happen directly in the syntax tree. -But the string replacement is slightly easier. Note that name spaces -are maintained in the C++ code to lessen the likelyhood of name -clashes. Currently no effort is made to detect name clashes. A good -rule of thumb is don't use values that start with '_' or 'py_' in -compiled expressions and you'll be fine. - -

Type definitions and coersion

-So far we've glossed over the dynamic vs. static typing issue between -Python and C++. In Python, the type of value that a variable holds can -change -through the course of program execution. C/C++, on the other hand, -forces you -to declare the type of value a variables will hold prior at compile -time. -weave.blitz handles this issue by examining the types of -the -variables in the expression being executed, and compiling a function -for those -explicit types. For example: -
- 

-    a = ones((5,5),Float32)
-    b = ones((5,5),Float32)
-    weave.blitz("a = a + b")
-
-
-When compiling this expression to C++, weave.blitz sees -that the -values for a and b in the local scope have type Float32, -or 'float' -on a 32 bit architecture. As a result, it compiles the function using -the float type (no attempt has been made to deal with 64 bit issues). -It also goes one step further. If all arrays have the same type, a -templated -version of the function is made and instantiated for float, double, -complex, and complex arrays. Note: This feature -has been removed from the current version of the code. Each version -will be compiled -separately - -

What happens if you call a compiled function with array types that -are different than the ones for which it was originally compiled? No -biggie, you'll just have to wait on it to compile a new version for -your new types. This doesn't overwrite the old functions, as they are -still accessible. See the catalog section in the inline() documentation -to see how this is handled. Suffice to say, the mechanism is -transparent to the user and behaves like dynamic typing with the -occasional wait for compiling newly typed functions. -

-

When working with combined scalar/array operations, the type of the -array is always used. This is similar to the savespace flag -that was recently added to NumPy. This prevents issues with the -following expression perhaps unexpectedly being calculated at a higher -(more expensive) precision that can occur in Python:

-
- 

-    >>> a = array((1,2,3),typecode = Float32)
-    >>> b = a * 2.1 # results in b being a Float64 array.
-
-
-In this example, -
- 

-    >>> a = ones((5,5),Float32)
-    >>> b = ones((5,5),Float32)
-    >>> weave.blitz("b = a * 2.1")
-
-
-the 2.1 is cast down to a float before -carrying out the operation. If you really want to force the calculation -to be a double, define a and b -as double arrays. -

One other point of note. Currently, you must include both the right -hand side and left hand side (assignment side) of your equation in the -compiled expression. Also, the array being assigned to must be created -prior to calling weave.blitz. I'm pretty sure this is -easily changed so that a compiled_eval expression can be defined, but -no effort has been made to allocate new arrays (and decern their type) -on the fly. -

-

Cataloging Compiled Functions

-See the Cataloging functions section in -the weave.inline() documentation. - -

Checking Array Sizes

-Surprisingly, one of the big initial problems with compiled code was -making -sure all the arrays in an operation were of compatible type. The -following -case is trivially easy: -
- 

-    a = b + c
-
-
-It only requires that arrays a, b, and c -have the same shape. However, expressions like: -
- 

-    a[i+j:i+j+1,:] = b[2:3,:] + c
-
-
-are not so trivial. Since slicing is involved, the size of the slices, -not the input arrays must be checked. Broadcasting complicates things -further because arrays and slices with different dimensions and shapes -may be compatible for math operations (broadcasting isn't yet supported -by weave.blitz). Reductions have a similar effect as -their results are different shapes than their input operand. The binary -operators in NumPy compare the shapes of their two operands just before -they operate on them. This is possible because NumPy treats each -operation independently. The intermediate (temporary) arrays created -during sub-operations in an expression are tested for the correct shape -before they are combined by another operation. Because weave.blitz -fuses all operations into a single loop, this isn't possible. The shape -comparisons must be done and guaranteed compatible before evaluating -the expression. -

The solution chosen converts input arrays to "dummy arrays" that -only represent the dimensions of the arrays, not the data. Binary -operations on dummy arrays check that input array sizes are comptible -and return a dummy array with the size correct size. Evaluating an -expression of dummy arrays traces the changing array sizes through all -operations and fails if incompatible array sizes are ever found.

-

The machinery for this is housed in weave.size_check. -It basically involves writing a new class (dummy array) and overloading -it math operators to calculate the new sizes correctly. All the code is -in Python and there is a fair amount of logic (mainly to handle -indexing and slicing) so the operation does impose some overhead. For -large arrays (ie. 50x50x50), the overhead is negligible compared to -evaluating the actual expression. For small arrays (ie. 16x16), the -overhead imposed for checking the shapes with this method can cause the -weave.blitz to be slower than evaluating the expression in -Python.

-

What can be done to reduce the overhead? (1) The size checking code -could be moved into C. This would likely remove most of the overhead -penalty compared to NumPy (although there is also some calling -overhead), but no effort has been made to do this. (2) You can also -call weave.blitz with -check_size=0 and the size checking isn't done. However, if -the sizes aren't compatible, it can cause a core-dump. So, foregoing -size_checking -isn't advisable until your code is well debugged. -

-

Creating the Extension Module

-weave.blitz uses the same machinery as weave.inline -to build the extension module. The only difference -is the code included in the function is automatically generated from -the NumPy array expression instead of supplied by the user. - -

Extension Modules

-weave.inline and weave.blitz are high level -tools -that generate extension modules automatically. Under the covers, they -use several -classes from weave.ext_tools to help generate the -extension module. -The main two classes are ext_module and ext_function -(I'd -like to add ext_class and ext_method also). -These classes -simplify the process of generating extension modules by handling most -of the "boiler -plate" code automatically. -Note: inline actually sub-classes weave.ext_tools.ext_function -to generate slightly different code than the standard ext_function. -The main difference is that the standard class converts function -arguments to -C types, while inline always has two arguments, the local and global -dicts, and -the grabs the variables that need to be convereted to C from these. - -

A Simple Example

-The following simple example demonstrates how to build an extension -module within -a Python function: -
- 

-    # examples/increment_example.py
-    from weave import ext_tools
-    
-    def build_increment_ext():
-        """ Build a simple extension with functions that increment numbers.
-            The extension will be built in the local directory.
-        """        
-        mod = ext_tools.ext_module('increment_ext')
-    
-        a = 1 # effectively a type declaration for 'a' in the 
-              # following functions.
-    
-        ext_code = "return_val = Py::new_reference_to(Py::Int(a+1));"    
-        func = ext_tools.ext_function('increment',ext_code,['a'])
-        mod.add_function(func)
-        
-        ext_code = "return_val = Py::new_reference_to(Py::Int(a+2));"    
-        func = ext_tools.ext_function('increment_by_2',ext_code,['a'])
-        mod.add_function(func)
-                
-        mod.compile()
-
-
-The function build_increment_ext() creates an extension -module named increment_ext and compiles it to a shared -library (.so or .pyd) that can be loaded into Python.. increment_ext -contains two functions, increment and increment_by_2. -The first line of build_increment_ext(), -
- 

-        mod = ext_tools.ext_module('increment_ext') 
-
-
-creates an ext_module instance that is ready to have ext_function -instances added to it. ext_function instances are created -much with a calling convention similar to weave.inline(). -The most common call includes a C/C++ code snippet and a list of the -arguments for the function. The following -
- 

-        ext_code = "return_val = Py::new_reference_to(Py::Int(a+1));"    
-        func = ext_tools.ext_function('increment',ext_code,['a'])
-
-
-creates a C/C++ extension function that is equivalent to the following -Python -function: -
- 

-        def increment(a):
-            return a + 1
-
-
-A second method is also added to the module and then, -
- 

-        mod.compile()
-
-
-is called to build the extension module. By default, the module is -created -in the current working directory. -This example is available in the examples/increment_example.py -file -found in the weave directory. At the bottom of the file -in the -module's "main" program, an attempt to import increment_ext -without -building it is made. If this fails (the module doesn't exist in the -PYTHONPATH), the module is built by calling build_increment_ext(). -This approach -only takes the time consuming ( a few seconds for this example) process -of building -the module if it hasn't been built before. -
- 

-    if __name__ == "__main__":
-        try:
-            import increment_ext
-        except ImportError:
-            build_increment_ext()
-            import increment_ext
-        a = 1
-        print 'a, a+1:', a, increment_ext.increment(a)
-        print 'a, a+2:', a, increment_ext.increment_by_2(a)           
-
-
-Note: If we were willing to always pay the penalty of building the -C++ code for a module, we could store the md5 checksum of the C++ code -along with some information about the compiler, platform, etc. Then, ext_module.compile() -could try importing the module before it actually -compiles it, check the md5 checksum and other meta-data in the imported -module -with the meta-data of the code it just produced and only compile the -code if -the module didn't exist or the meta-data didn't match. This would -reduce the -above code to: - -
- 

-    if __name__ == "__main__":
-        build_increment_ext()
-
-        a = 1
-        print 'a, a+1:', a, increment_ext.increment(a)
-        print 'a, a+2:', a, increment_ext.increment_by_2(a)           
-
-
-Note: There would always be the overhead of building the C++ code, -but it would only actually compile the code once. You pay a little in -overhead and get cleaner -"import" code. Needs some thought. - -

If you run increment_example.py from the command line, -you get -the following:

-
- 

-    [eric at n0]$ python increment_example.py
-    a, a+1: 1 2
-    a, a+2: 1 3
-
-
-If the module didn't exist before it was run, the module is created. If -it did -exist, it is just imported and used. - -

Fibonacci Example

-examples/fibonacci.py provides a little more complex -example of how to use ext_tools. Fibonacci numbers are a -series of numbers where each number in the series is the sum of the -previous two: 1, 1, 2, 3, 5, 8, etc. Here, the first two numbers in the -series are taken to be 1. One approach to calculating Fibonacci numbers -uses recursive function calls. In Python, it might be written as: -
- 

-    def fib(a):
-        if a <= 2:
-            return 1
-        else:
-            return fib(a-2) + fib(a-1)
-
-
-In C, the same function would look something like this: -
- 

-     int fib(int a)
-     {                   
-         if(a <= 2)
-             return 1;
-         else
-             return fib(a-2) + fib(a-1);  
-     }                      
-
-
-Recursion is much faster in C than in Python, so it would be beneficial -to use the C version for fibonacci number calculations instead of the -Python version. We need an extension function that calls this C -function -to do this. This is possible by including the above code snippet as -"support code" and then calling it from the extension function. Support -code snippets (usually structure definitions, helper functions and the -like) -are inserted into the extension module C/C++ file before the extension -function code. Here is how to build the C version of the fibonacci -number -generator: -
- 

-def build_fibonacci():
-    """ Builds an extension module with fibonacci calculators.
-    """
-    mod = ext_tools.ext_module('fibonacci_ext')
-    a = 1 # this is effectively a type declaration
-    
-    # recursive fibonacci in C 
-    fib_code = """
-                   int fib1(int a)
-                   {                   
-                       if(a <= 2)
-                           return 1;
-                       else
-                           return fib1(a-2) + fib1(a-1);  
-                   }                         
-               """
-    ext_code = """
-                   int val = fib1(a);
-                   return_val = Py::new_reference_to(Py::Int(val));
-               """    
-    fib = ext_tools.ext_function('fib',ext_code,['a'])
-    fib.customize.add_support_code(fib_code)
-    mod.add_function(fib)
-
-    mod.compile()
-
-
-
-XXX More about custom_info, and what xxx_info instances are good for. -

Note: recursion is not the fastest way to calculate fibonacci -numbers, -but this approach serves nicely for this example. -

-

-

-

Customizing Type Conversions -- Type Factories

-not written -

Things I wish weave did

-It is possible to get name clashes if you uses a variable name that is -already defined -in a header automatically included (such as stdio.h) For -instance, if you -try to pass in a variable named stdout, you'll get a -cryptic error report -due to the fact that stdio.h also defines the name. weave -should probably try and handle this in some way. -Other things... - - Added: trunk/scipy/weave/doc/tutorial.txt =================================================================== --- trunk/scipy/weave/doc/tutorial.txt 2007-10-24 22:55:18 UTC (rev 3461) +++ trunk/scipy/weave/doc/tutorial.txt 2007-10-25 17:09:53 UTC (rev 3462) @@ -0,0 +1,2531 @@ +===================== + Weave Documentation +===================== + +By Eric Jones eric at enthought.com + + +Outline +======= + +.. contents:: + + +============== + Introduction +============== + +The ``weave`` package provides tools for including C/C++ code within in +Python code. This offers both another level of optimization to those who need +it, and an easy way to modify and extend any supported extension libraries +such as wxPython and hopefully VTK soon. Inlining C/C++ code within Python +generally results in speed ups of 1.5x to 30x speed-up over algorithms +written in pure Python (However, it is also possible to slow things down...). +Generally algorithms that require a large number of calls to the Python API +don't benefit as much from the conversion to C/C++ as algorithms that have +inner loops completely convertable to C. + +There are three basic ways to use ``weave``. The ``weave.inline()`` function +executes C code directly within Python, and ``weave.blitz()`` translates +Python NumPy expressions to C++ for fast execution. ``blitz()`` was the +original reason ``weave`` was built. For those interested in building +extension libraries, the ``ext_tools`` module provides classes for building +extension modules within Python. + +Most of ``weave's`` functionality should work on Windows and Unix, although +some of its functionality requires ``gcc`` or a similarly modern C++ compiler +that handles templates well. Up to now, most testing has been done on Windows +2000 with Microsoft's C++ compiler (MSVC) and with gcc (mingw32 2.95.2 and +2.95.3-6). All tests also pass on Linux (RH 7.1 with gcc 2.96), and I've had +reports that it works on Debian also (thanks Pearu). + +The ``inline`` and ``blitz`` provide new functionality to Python (although +I've recently learned about the `PyInline`_ project which may offer similar +functionality to ``inline``). On the other hand, tools for building Python +extension modules already exists (SWIG, SIP, pycpp, CXX, and others). As of +yet, I'm not sure where ``weave`` fits in this spectrum. It is closest in +flavor to CXX in that it makes creating new C/C++ extension modules pretty +easy. However, if you're wrapping a gaggle of legacy functions or classes, +SWIG and friends are definitely the better choice. ``weave`` is set up so +that you can customize how Python types are converted to C types in +``weave``. This is great for ``inline()``, but, for wrapping legacy code, it +is more flexible to specify things the other way around -- that is how C +types map to Python types. This ``weave`` does not do. I guess it would be +possible to build such a tool on top of ``weave``, but with good tools like +SWIG around, I'm not sure the effort produces any new capabilities. Things +like function overloading are probably easily implemented in ``weave`` and it +might be easier to mix Python/C code in function calls, but nothing beyond +this comes to mind. So, if you're developing new extension modules or +optimizing Python functions in C, ``weave.ext_tools()`` might be the tool for +you. If you're wrapping legacy code, stick with SWIG. + +The next several sections give the basics of how to use ``weave``. We'll +discuss what's happening under the covers in more detail later on. Serious +users will need to at least look at the type conversion section to understand +how Python variables map to C/C++ types and how to customize this behavior. +One other note. If you don't know C or C++ then these docs are probably of +very little help to you. Further, it'd be helpful if you know something about +writing Python extensions. ``weave`` does quite a bit for you, but for +anything complex, you'll need to do some conversions, reference counting, +etc. + +.. note:: + ``weave`` is actually part of the `SciPy`_ package. However, it + also works fine as a standalone package (you can check out the sources using + ``svn co http://svn.scipy.org/svn/scipy/trunk/Lib/weave weave`` and install as + python setup.py install). The examples here are given as if it is used as a + stand alone package. If you are using from within scipy, you can use `` from + scipy import weave`` and the examples will work identically. + + +============== + Requirements +============== + +- Python + + I use 2.1.1. Probably 2.0 or higher should work. + +- C++ compiler + + ``weave`` uses ``distutils`` to actually build extension modules, so + it uses whatever compiler was originally used to build Python. ``weave`` + itself requires a C++ compiler. If you used a C++ compiler to build + Python, your probably fine. + + On Unix gcc is the preferred choice because I've done a little + testing with it. All testing has been done with gcc, but I expect the + majority of compilers should work for ``inline`` and ``ext_tools``. The + one issue I'm not sure about is that I've hard coded things so that + compilations are linked with the ``stdc++`` library. *Is this standard + across Unix compilers, or is this a gcc-ism?* + + For ``blitz()``, you'll need a reasonably recent version of gcc. + 2.95.2 works on windows and 2.96 looks fine on Linux. Other versions are + likely to work. Its likely that KAI's C++ compiler and maybe some others + will work, but I haven't tried. My advice is to use gcc for now unless + your willing to tinker with the code some. + + On Windows, either MSVC or gcc (`mingw32`_) should work. Again, + you'll need gcc for ``blitz()`` as the MSVC compiler doesn't handle + templates well. + + I have not tried Cygwin, so please report success if it works for + you. + +- NumPy + + The python `NumPy`_ module is required for ``blitz()`` to + work and for numpy.distutils which is used by weave. + + +============== + Installation +============== + +There are currently two ways to get ``weave``. First, ``weave`` is part of +SciPy and installed automatically (as a sub- package) whenever SciPy is +installed. Second, since ``weave`` is useful outside of the scientific +community, it has been setup so that it can be used as a stand-alone module. + +The stand-alone version can be downloaded from `here`_. Instructions for +installing should be found there as well. setup.py file to simplify +installation. + + +========= + Testing +========= + +Once ``weave`` is installed, fire up python and run its unit tests. + +:: + + >>> import weave + >>> weave.test() + runs long time... spews tons of output and a few warnings + . + . + . + .............................................................. + ................................................................ + .................................................. + ---------------------------------------------------------------------- + Ran 184 tests in 158.418s + OK + >>> + + +This takes a while, usually several minutes. On Unix with remote file +systems, I've had it take 15 or so minutes. In the end, it should run about +180 tests and spew some speed results along the way. If you get errors, +they'll be reported at the end of the output. Please report errors that you +find. Some tests are known to fail at this point. + + +If you only want to test a single module of the package, you can do this by +running test() for that specific module. + +:: + + >>> import weave.scalar_spec + >>> weave.scalar_spec.test() + ....... + ---------------------------------------------------------------------- + Ran 7 tests in 23.284s + + +Testing Notes: +============== + + +- Windows 1 + + I've had some test fail on windows machines where I have msvc, + gcc-2.95.2 (in c:\gcc-2.95.2), and gcc-2.95.3-6 (in c:\gcc) all + installed. My environment has c:\gcc in the path and does not have + c:\gcc-2.95.2 in the path. The test process runs very smoothly until the + end where several test using gcc fail with cpp0 not found by g++. If I + check os.system('gcc -v') before running tests, I get gcc-2.95.3-6. If I + check after running tests (and after failure), I get gcc-2.95.2. ??huh??. + The os.environ['PATH'] still has c:\gcc first in it and is not corrupted + (msvc/distutils messes with the environment variables, so we have to undo + its work in some places). If anyone else sees this, let me know - - it + may just be an quirk on my machine (unlikely). Testing with the gcc- + 2.95.2 installation always works. + +- Windows 2 + + If you run the tests from PythonWin or some other GUI tool, you'll + get a ton of DOS windows popping up periodically as ``weave`` spawns the + compiler multiple times. Very annoying. Anyone know how to fix this? + +- wxPython + + wxPython tests are not enabled by default because importing wxPython + on a Unix machine without access to a X-term will cause the program to + exit. Anyone know of a safe way to detect whether wxPython can be + imported and whether a display exists on a machine? + +============ + Benchmarks +============ + +This section has not been updated from old scipy weave and Numeric.... + +This section has a few benchmarks -- thats all people want to see anyway +right? These are mostly taken from running files in the ``weave/example`` +directory and also from the test scripts. Without more information about what +the test actually do, their value is limited. Still, their here for the +curious. Look at the example scripts for more specifics about what problem +was actually solved by each run. These examples are run under windows 2000 +using Microsoft Visual C++ and python2.1 on a 850 MHz PIII laptop with 320 MB +of RAM. Speed up is the improvement (degredation) factor of ``weave`` +compared to conventional Python functions. ``The blitz()`` comparisons are +shown compared to NumPy. + +inline and ext_tools + +Algorithm + +Speed up + +binary search 1.50 +fibonacci (recursive) 82.10 +fibonacci (loop) 9.17 +return None 0.14 +map 1.20 +dictionary sort 2.54 +vector quantization 37.40 + +blitz -- double precision + +Algorithm + +Speed up + +a = b + c 512x512 3.05 +a = b + c + d 512x512 4.59 +5 pt avg. filter, 2D Image 512x512 9.01 +Electromagnetics (FDTD) 100x100x100 8.61 + +The benchmarks shown ``blitz`` in the best possible light. NumPy (at least on +my machine) is significantly worse for double precision than it is for single +precision calculations. If your interested in single precision results, you +can pretty much divide the double precision speed up by 3 and you'll be +close. + + +======== + Inline +======== + +``inline()`` compiles and executes C/C++ code on the fly. Variables in the +local and global Python scope are also available in the C/C++ code. Values +are passed to the C/C++ code by assignment much like variables are passed +into a standard Python function. Values are returned from the C/C++ code +through a special argument called return_val. Also, the contents of mutable +objects can be changed within the C/C++ code and the changes remain after the +C code exits and returns to Python. (more on this later) + +Here's a trivial ``printf`` example using ``inline()``:: + + >>> import weave + >>> a = 1 + >>> weave.inline('printf("%d\\n",a);',['a']) + 1 + +In this, its most basic form, ``inline(c_code, var_list)`` requires two +arguments. ``c_code`` is a string of valid C/C++ code. ``var_list`` is a list +of variable names that are passed from Python into C/C++. Here we have a +simple ``printf`` statement that writes the Python variable ``a`` to the +screen. The first time you run this, there will be a pause while the code is +written to a .cpp file, compiled into an extension module, loaded into +Python, cataloged for future use, and executed. On windows (850 MHz PIII), +this takes about 1.5 seconds when using Microsoft's C++ compiler (MSVC) and +6-12 seconds using gcc (mingw32 2.95.2). All subsequent executions of the +code will happen very quickly because the code only needs to be compiled +once. If you kill and restart the interpreter and then execute the same code +fragment again, there will be a much shorter delay in the fractions of +seconds range. This is because ``weave`` stores a catalog of all previously +compiled functions in an on disk cache. When it sees a string that has been +compiled, it loads the already compiled module and executes the appropriate +function. + +.. note:: + If you try the ``printf`` example in a GUI shell such as IDLE, + PythonWin, PyShell, etc., you're unlikely to see the output. This is because + the C code is writing to stdout, instead of to the GUI window. This doesn't + mean that inline doesn't work in these environments -- it only means that + standard out in C is not the same as the standard out for Python in these + cases. Non input/output functions will work as expected. + +Although effort has been made to reduce the overhead associated with calling +inline, it is still less efficient for simple code snippets than using +equivalent Python code. The simple ``printf`` example is actually slower by +30% or so than using Python ``print`` statement. And, it is not difficult to +create code fragments that are 8-10 times slower using inline than equivalent +Python. However, for more complicated algorithms, the speed up can be worth +while -- anywhwere from 1.5- 30 times faster. Algorithms that have to +manipulate Python objects (sorting a list) usually only see a factor of 2 or +so improvement. Algorithms that are highly computational or manipulate NumPy +arrays can see much larger improvements. The examples/vq.py file shows a +factor of 30 or more improvement on the vector quantization algorithm that is +used heavily in information theory and classification problems. + + +More with printf +================ + +MSVC users will actually see a bit of compiler output that distutils does not +supress the first time the code executes:: + + >>> weave.inline(r'printf("%d\n",a);',['a']) + sc_e013937dbc8c647ac62438874e5795131.cpp + Creating library C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp + \Release\sc_e013937dbc8c647ac62438874e5795131.lib and + object C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_e013937dbc8c647ac62438874e5795131.exp + 1 + +Nothing bad is happening, its just a bit annoying. * Anyone know how to turn +this off?* + +This example also demonstrates using 'raw strings'. The ``r`` preceeding the +code string in the last example denotes that this is a 'raw string'. In raw +strings, the backslash character is not interpreted as an escape character, +and so it isn't necessary to use a double backslash to indicate that the '\n' +is meant to be interpreted in the C ``printf`` statement instead of by +Python. If your C code contains a lot of strings and control characters, raw +strings might make things easier. Most of the time, however, standard strings +work just as well. + +The ``printf`` statement in these examples is formatted to print out +integers. What happens if ``a`` is a string? ``inline`` will happily, compile +a new version of the code to accept strings as input, and execute the code. +The result? + +:: + + >>> a = 'string' + >>> weave.inline(r'printf("%d\n",a);',['a']) + 32956972 + + +In this case, the result is non-sensical, but also non-fatal. In other +situations, it might produce a compile time error because ``a`` is required +to be an integer at some point in the code, or it could produce a +segmentation fault. Its possible to protect against passing ``inline`` +arguments of the wrong data type by using asserts in Python. + +:: + + >>> a = 'string' + >>> def protected_printf(a): + ... assert(type(a) == type(1)) + ... weave.inline(r'printf("%d\n",a);',['a']) + >>> protected_printf(1) + 1 + >>> protected_printf('string') + AssertError... + + +For printing strings, the format statement needs to be changed. Also, weave +doesn't convert strings to char*. Instead it uses CXX Py::String type, so you +have to do a little more work. Here we convert it to a C++ std::string and +then ask cor the char* version. + +:: + + >>> a = 'string' + >>> weave.inline(r'printf("%s\n",std::string(a).c_str());',['a']) + string + +.. admonition:: XXX + + This is a little convoluted. Perhaps strings should convert to ``std::string`` + objects instead of CXX objects. Or maybe to ``char*``. + +As in this case, C/C++ code fragments often have to change to accept +different types. For the given printing task, however, C++ streams provide a +way of a single statement that works for integers and strings. By default, +the stream objects live in the std (standard) namespace and thus require the +use of ``std::``. + +:: + + >>> weave.inline('std::cout << a << std::endl;',['a']) + 1 + >>> a = 'string' + >>> weave.inline('std::cout << a << std::endl;',['a']) + string + + +Examples using ``printf`` and ``cout`` are included in +examples/print_example.py. + + +More examples +============= + +This section shows several more advanced uses of ``inline``. It includes a +few algorithms from the `Python Cookbook`_ that have been re-written in +inline C to improve speed as well as a couple examples using NumPy and +wxPython. + +Binary search +------------- + +Lets look at the example of searching a sorted list of integers for a value. +For inspiration, we'll use Kalle Svensson's `binary_search()`_ algorithm +from the Python Cookbook. His recipe follows:: + + def binary_search(seq, t): + min = 0; max = len(seq) - 1 + while 1: + if max < min: + return -1 + m = (min + max) / 2 + if seq[m] < t: + min = m + 1 + elif seq[m] > t: + max = m - 1 + else: + return m + + +This Python version works for arbitrary Python data types. The C version +below is specialized to handle integer values. There is a little type +checking done in Python to assure that we're working with the correct data +types before heading into C. The variables ``seq`` and ``t`` don't need to be +declared beacuse ``weave`` handles converting and declaring them in the C +code. All other temporary variables such as ``min, max``, etc. must be +declared -- it is C after all. Here's the new mixed Python/C function:: + + def c_int_binary_search(seq,t): + # do a little type checking in Python + assert(type(t) == type(1)) + assert(type(seq) == type([])) + + # now the C code + code = """ + #line 29 "binary_search.py" + int val, m, min = 0; + int max = seq.length() - 1; + PyObject *py_val; + for(;;) + { + if (max < min ) + { + return_val = Py::new_reference_to(Py::Int(-1)); + break; + } + m = (min + max) /2; + val = py_to_int(PyList_GetItem(seq.ptr(),m),"val"); + if (val < t) + min = m + 1; + else if (val > t) + max = m - 1; + else + { + return_val = Py::new_reference_to(Py::Int(m)); + break; + } + } + """ + return inline(code,['seq','t']) + +We have two variables ``seq`` and ``t`` passed in. ``t`` is guaranteed (by +the ``assert``) to be an integer. Python integers are converted to C int +types in the transition from Python to C. ``seq`` is a Python list. By +default, it is translated to a CXX list object. Full documentation for the +CXX library can be found at its `website`_. The basics are that the CXX +provides C++ class equivalents for Python objects that simplify, or at least +object orientify, working with Python objects in C/C++. For example, +``seq.length()`` returns the length of the list. A little more about CXX and +its class methods, etc. is in the ** type conversions ** section. + +.. note:: + CXX uses templates and therefore may be a little less portable than + another alternative by Gordan McMillan called SCXX which was + inspired by CXX. It doesn't use templates so it should compile + faster and be more portable. SCXX has a few less features, but it + appears to me that it would mesh with the needs of weave quite well. + Hopefully xxx_spec files will be written for SCXX in the future, and + we'll be able to compare on a more empirical basis. Both sets of + spec files will probably stick around, it just a question of which + becomes the default. + +Most of the algorithm above looks similar in C to the original Python code. +There are two main differences. The first is the setting of ``return_val`` +instead of directly returning from the C code with a ``return`` statement. +``return_val`` is an automatically defined variable of type ``PyObject*`` +that is returned from the C code back to Python. You'll have to handle +reference counting issues when setting this variable. In this example, CXX +classes and functions handle the dirty work. All CXX functions and classes +live in the namespace ``Py::``. The following code converts the integer ``m`` +to a CXX ``Int()`` object and then to a ``PyObject*`` with an incremented +reference count using ``Py::new_reference_to()``. + +:: + return_val = Py::new_reference_to(Py::Int(m)); + + +The second big differences shows up in the retrieval of integer values from +the Python list. The simple Python ``seq[i]`` call balloons into a C Python +API call to grab the value out of the list and then a separate call to +``py_to_int()`` that converts the PyObject* to an integer. ``py_to_int()`` +includes both a NULL cheack and a ``PyInt_Check()`` call as well as the +conversion call. If either of the checks fail, an exception is raised. The +entire C++ code block is executed with in a ``try/catch`` block that handles +exceptions much like Python does. This removes the need for most error +checking code. + +It is worth note that CXX lists do have indexing operators that result in +code that looks much like Python. However, the overhead in using them appears +to be relatively high, so the standard Python API was used on the +``seq.ptr()`` which is the underlying ``PyObject*`` of the List object. + +The ``#line`` directive that is the first line of the C code block isn't +necessary, but it's nice for debugging. If the compilation fails because of +the syntax error in the code, the error will be reported as an error in the +Python file "binary_search.py" with an offset from the given line number (29 +here). + +So what was all our effort worth in terms of efficiency? Well not a lot in +this case. The examples/binary_search.py file runs both Python and C versions +of the functions As well as using the standard ``bisect`` module. If we run +it on a 1 million element list and run the search 3000 times (for 0- 2999), +here are the results we get:: + + C:\home\ej\wrk\scipy\weave\examples> python binary_search.py + Binary search for 3000 items in 1000000 length list of integers: + speed in python: 0.159999966621 + speed of bisect: 0.121000051498 + speed up: 1.32 + speed in c: 0.110000014305 + speed up: 1.45 + speed in c(no asserts): 0.0900000333786 + speed up: 1.78 + + +So, we get roughly a 50-75% improvement depending on whether we use the +Python asserts in our C version. If we move down to searching a 10000 element +list, the advantage evaporates. Even smaller lists might result in the Python +version being faster. I'd like to say that moving to NumPy lists (and getting +rid of the GetItem() call) offers a substantial speed up, but my preliminary +efforts didn't produce one. I think the log(N) algorithm is to blame. Because +the algorithm is nice, there just isn't much time spent computing things, so +moving to C isn't that big of a win. If there are ways to reduce conversion +overhead of values, this may improve the C/Python speed up. Anyone have other +explanations or faster code, please let me know. + + +Dictionary Sort +--------------- + +The demo in examples/dict_sort.py is another example from the Python +CookBook. `This submission`_, by Alex Martelli, demonstrates how to return +the values from a dictionary sorted by their keys: + +:: + def sortedDictValues3(adict): + keys = adict.keys() + keys.sort() + return map(adict.get, keys) + + +Alex provides 3 algorithms and this is the 3rd and fastest of the set. The C +version of this same algorithm follows:: + + def c_sort(adict): + assert(type(adict) == type({})) + code = """ + #line 21 "dict_sort.py" + Py::List keys = adict.keys(); + Py::List items(keys.length()); keys.sort(); + PyObject* item = NULL; + for(int i = 0; i < keys.length();i++) + { + item = PyList_GET_ITEM(keys.ptr(),i); + item = PyDict_GetItem(adict.ptr(),item); + Py_XINCREF(item); + PyList_SetItem(items.ptr(),i,item); + } + return_val = Py::new_reference_to(items); + """ + return inline_tools.inline(code,['adict'],verbose=1) + + +Like the original Python function, the C++ version can handle any Python +dictionary regardless of the key/value pair types. It uses CXX objects for +the most part to declare python types in C++, but uses Python API calls to +manipulate their contents. Again, this choice is made for speed. The C++ +version, while more complicated, is about a factor of 2 faster than Python. + +:: + + C:\home\ej\wrk\scipy\weave\examples> python dict_sort.py + Dict sort of 1000 items for 300 iterations: + speed in python: 0.319999933243 + [0, 1, 2, 3, 4] + speed in c: 0.151000022888 + speed up: 2.12 + [0, 1, 2, 3, 4] + + + +NumPy -- cast/copy/transpose +---------------------------- + +CastCopyTranspose is a function called quite heavily by Linear Algebra +routines in the NumPy library. Its needed in part because of the row-major +memory layout of multi-demensional Python (and C) arrays vs. the col-major +order of the underlying Fortran algorithms. For small matrices (say 100x100 +or less), a significant portion of the common routines such as LU +decompisition or singular value decompostion are spent in this setup routine. +This shouldn't happen. Here is the Python version of the function using +standard NumPy operations. + +:: + + def _castCopyAndTranspose(type, array): + if a.typecode() == type: + cast_array = copy.copy(NumPy.transpose(a)) + else: + cast_array = copy.copy(NumPy.transpose(a).astype(type)) + return cast_array + + +And the following is a inline C version of the same function:: + + from weave.blitz_tools import blitz_type_factories + from weave import scalar_spec + from weave import inline + def _cast_copy_transpose(type,a_2d): + assert(len(shape(a_2d)) == 2) + new_array = zeros(shape(a_2d),type) + NumPy_type = scalar_spec.NumPy_to_blitz_type_mapping[type] + code = \ + """ + for(int i = 0;i < _Na_2d[0]; i++) + for(int j = 0; j < _Na_2d[1]; j++) + new_array(i,j) = (%s) a_2d(j,i); + """ % NumPy_type + inline(code,['new_array','a_2d'], + type_factories = blitz_type_factories,compiler='gcc') + return new_array + + +This example uses blitz++ arrays instead of the standard representation of +NumPy arrays so that indexing is simplier to write. This is accomplished by +passing in the blitz++ "type factories" to override the standard Python to +C++ type conversions. Blitz++ arrays allow you to write clean, fast code, but +they also are sloooow to compile (20 seconds or more for this snippet). This +is why they aren't the default type used for Numeric arrays (and also because +most compilers can't compile blitz arrays...). ``inline()`` is also forced to +use 'gcc' as the compiler because the default compiler on Windows (MSVC) will +not compile blitz code. ('gcc' I think will use the standard compiler on +Unix machine instead of explicitly forcing gcc (check this)) Comparisons of +the Python vs inline C++ code show a factor of 3 speed up. Also shown are the +results of an "inplace" transpose routine that can be used if the output of +the linear algebra routine can overwrite the original matrix (this is often +appropriate). This provides another factor of 2 improvement. + +:: + + #C:\home\ej\wrk\scipy\weave\examples> python cast_copy_transpose.py + # Cast/Copy/Transposing (150,150)array 1 times + # speed in python: 0.870999932289 + # speed in c: 0.25 + # speed up: 3.48 + # inplace transpose c: 0.129999995232 + # speed up: 6.70 + +wxPython +-------- + +``inline`` knows how to handle wxPython objects. Thats nice in and of itself, +but it also demonstrates that the type conversion mechanism is reasonably +flexible. Chances are, it won't take a ton of effort to support special types +you might have. The examples/wx_example.py borrows the scrolled window +example from the wxPython demo, accept that it mixes inline C code in the +middle of the drawing function. + +:: + + def DoDrawing(self, dc): + + red = wxNamedColour("RED"); + blue = wxNamedColour("BLUE"); + grey_brush = wxLIGHT_GREY_BRUSH; + code = \ + """ + #line 108 "wx_example.py" + dc->BeginDrawing(); + dc->SetPen(wxPen(*red,4,wxSOLID)); + dc->DrawRectangle(5,5,50,50); + dc->SetBrush(*grey_brush); + dc->SetPen(wxPen(*blue,4,wxSOLID)); + dc->DrawRectangle(15, 15, 50, 50); + """ + inline(code,['dc','red','blue','grey_brush']) + + dc.SetFont(wxFont(14, wxSWISS, wxNORMAL, wxNORMAL)) + dc.SetTextForeground(wxColour(0xFF, 0x20, 0xFF)) + te = dc.GetTextExtent("Hello World") + dc.DrawText("Hello World", 60, 65) + + dc.SetPen(wxPen(wxNamedColour('VIOLET'), 4)) + dc.DrawLine(5, 65+te[1], 60+te[0], 65+te[1]) + ... + +Here, some of the Python calls to wx objects were just converted to C++ +calls. There isn't any benefit, it just demonstrates the capabilities. You +might want to use this if you have a computationally intensive loop in your +drawing code that you want to speed up. On windows, you'll have to use the +MSVC compiler if you use the standard wxPython DLLs distributed by Robin +Dunn. Thats because MSVC and gcc, while binary compatible in C, are not +binary compatible for C++. In fact, its probably best, no matter what +platform you're on, to specify that ``inline`` use the same compiler that was +used to build wxPython to be on the safe side. There isn't currently a way to +learn this info from the library -- you just have to know. Also, at least on +the windows platform, you'll need to install the wxWindows libraries and link +to them. I think there is a way around this, but I haven't found it yet -- I +get some linking errors dealing with wxString. One final note. You'll +probably have to tweak weave/wx_spec.py or weave/wx_info.py for your +machine's configuration to point at the correct directories etc. There. That +should sufficiently scare people into not even looking at this... :) + +Keyword Option +============== + +The basic definition of the ``inline()`` function has a slew of optional +variables. It also takes keyword arguments that are passed to ``distutils`` +as compiler options. The following is a formatted cut/paste of the argument +section of ``inline's`` doc-string. It explains all of the variables. Some +examples using various options will follow. + +:: + + def inline(code,arg_names,local_dict = None, global_dict = None, + force = 0, + compiler='', + verbose = 0, + support_code = None, + customize=None, + type_factories = None, + auto_downcast=1, + **kw): + + +``inline`` has quite a few options as listed below. Also, the keyword +arguments for distutils extension modules are accepted to specify extra +information needed for compiling. + +Inline Arguments +================ + +code string. A string of valid C++ code. It should not specify a return +statement. Instead it should assign results that need to be returned to +Python in the return_val. arg_names list of strings. A list of Python +variable names that should be transferred from Python into the C/C++ code. +local_dict optional. dictionary. If specified, it is a dictionary of values +that should be used as the local scope for the C/C++ code. If local_dict is +not specified the local dictionary of the calling function is used. +global_dict optional. dictionary. If specified, it is a dictionary of values +that should be used as the global scope for the C/C++ code. If global_dict is +not specified the global dictionary of the calling function is used. force +optional. 0 or 1. default 0. If 1, the C++ code is compiled every time inline +is called. This is really only useful for debugging, and probably only useful +if you're editing support_code a lot. compiler optional. string. The name +of compiler to use when compiling. On windows, it understands 'msvc' and +'gcc' as well as all the compiler names understood by distutils. On Unix, +it'll only understand the values understoof by distutils. (I should add 'gcc' +though to this). + +On windows, the compiler defaults to the Microsoft C++ compiler. If this +isn't available, it looks for mingw32 (the gcc compiler). + +On Unix, it'll probably use the same compiler that was used when compiling +Python. Cygwin's behavior should be similar. + +verbose optional. 0,1, or 2. defualt 0. Speficies how much much +information is printed during the compile phase of inlining code. 0 is silent +(except on windows with msvc where it still prints some garbage). 1 informs +you when compiling starts, finishes, and how long it took. 2 prints out the +command lines for the compilation process and can be useful if you're having +problems getting code to work. Its handy for finding the name of the .cpp +file if you need to examine it. verbose has no affect if the compilation +isn't necessary. support_code optional. string. A string of valid C++ code +declaring extra code that might be needed by your compiled function. This +could be declarations of functions, classes, or structures. customize +optional. base_info.custom_info object. An alternative way to specifiy +support_code, headers, etc. needed by the function see the weave.base_info +module for more details. (not sure this'll be used much). type_factories +optional. list of type specification factories. These guys are what convert +Python data types to C/C++ data types. If you'd like to use a different set +of type conversions than the default, specify them here. Look in the type +conversions section of the main documentation for examples. auto_downcast +optional. 0 or 1. default 1. This only affects functions that have Numeric +arrays as input variables. Setting this to 1 will cause all floating point +values to be cast as float instead of double if all the NumPy arrays are of +type float. If even one of the arrays has type double or double complex, all +variables maintain there standard types. + + +Distutils keywords +================== + +``inline()`` also accepts a number of ``distutils`` keywords for +controlling how the code is compiled. The following descriptions have been +copied from Greg Ward's ``distutils.extension.Extension`` class doc- strings +for convenience: sources [string] list of source filenames, relative to the +distribution root (where the setup script lives), in Unix form (slash- +separated) for portability. Source files may be C, C++, SWIG (.i), platform- +specific resource files, or whatever else is recognized by the "build_ext" +command as source for a Python extension. Note: The module_path file is +always appended to the front of this list include_dirs [string] list of +directories to search for C/C++ header files (in Unix form for portability) +define_macros [(name : string, value : string|None)] list of macros to +define; each macro is defined using a 2-tuple, where 'value' is either the +string to define it to or None to define it without a particular value +(equivalent of "#define FOO" in source or -DFOO on Unix C compiler command +line) undef_macros [string] list of macros to undefine explicitly +library_dirs [string] list of directories to search for C/C++ libraries at +link time libraries [string] list of library names (not filenames or paths) +to link against runtime_library_dirs [string] list of directories to search +for C/C++ libraries at run time (for shared extensions, this is when the +extension is loaded) extra_objects [string] list of extra files to link +with (eg. object files not implied by 'sources', static library that must be +explicitly specified, binary resource files, etc.) extra_compile_args +[string] any extra platform- and compiler-specific information to use when +compiling the source files in 'sources'. For platforms and compilers where +"command line" makes sense, this is typically a list of command-line +arguments, but for other platforms it could be anything. extra_link_args +[string] any extra platform- and compiler-specific information to use when +linking object files together to create the extension (or to create a new +static Python interpreter). Similar interpretation as for +'extra_compile_args'. export_symbols [string] list of symbols to be +exported from a shared extension. Not used on all platforms, and not +generally necessary for Python extensions, which typically export exactly one +symbol: "init" + extension_name. + + +Keyword Option Examples +----------------------- + +We'll walk through several examples here to demonstrate the behavior of +``inline`` and also how the various arguments are used. In the simplest +(most) cases, ``code`` and ``arg_names`` are the only arguments that need to +be specified. Here's a simple example run on Windows machine that has +Microsoft VC++ installed. + +:: + + >>> from weave import inline + >>> a = 'string' + >>> code = """ + ... int l = a.length(); + ... return_val = Py::new_reference_to(Py::Int(l)); + ... """ + >>> inline(code,['a']) + sc_86e98826b65b047ffd2cd5f479c627f12.cpp + Creating + library C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ffd2cd5f479c627f12.lib + and object C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ff + d2cd5f479c627f12.exp + 6 + >>> inline(code,['a']) + 6 + + +When ``inline`` is first run, you'll notice that pause and some trash printed +to the screen. The "trash" is acutually part of the compilers output that +distutils does not supress. The name of the extension file, +``sc_bighonkingnumber.cpp``, is generated from the md5 check sum of the C/C++ +code fragment. On Unix or windows machines with only gcc installed, the trash +will not appear. On the second call, the code fragment is not compiled since +it already exists, and only the answer is returned. Now kill the interpreter +and restart, and run the same code with a different string. + +:: + + >>> from weave import inline + >>> a = 'a longer string' + >>> code = """ + ... int l = a.length(); + ... return_val = Py::new_reference_to(Py::Int(l)); + ... """ + >>> inline(code,['a']) + 15 + + +Notice this time, ``inline()`` did not recompile the code because it found +the compiled function in the persistent catalog of functions. There is a +short pause as it looks up and loads the function, but it is much shorter +than compiling would require. + +You can specify the local and global dictionaries if you'd like (much like +``exec`` or ``eval()`` in Python), but if they aren't specified, the +"expected" ones are used -- i.e. the ones from the function that called +``inline()``. This is accomplished through a little call frame trickery. +Here is an example where the local_dict is specified using the same code +example from above:: + + >>> a = 'a longer string' + >>> b = 'an even longer string' + >>> my_dict = {'a':b} + >>> inline(code,['a']) + 15 + >>> inline(code,['a'],my_dict) + 21 + + +Everytime, the ``code`` is changed, ``inline`` does a recompile. However, +changing any of the other options in inline does not force a recompile. The +``force`` option was added so that one could force a recompile when tinkering +with other variables. In practice, it is just as easy to change the ``code`` +by a single character (like adding a space some place) to force the +recompile. + +.. note:: + It also might be nice to add some methods for purging the + cache and on disk catalogs. + +I use ``verbose`` sometimes for debugging. When set to 2, it'll output all +the information (including the name of the .cpp file) that you'd expect from +running a make file. This is nice if you need to examine the generated code +to see where things are going haywire. Note that error messages from failed +compiles are printed to the screen even if ``verbose`` is set to 0. + +The following example demonstrates using gcc instead of the standard msvc +compiler on windows using same code fragment as above. Because the example +has already been compiled, the ``force=1`` flag is needed to make +``inline()`` ignore the previously compiled version and recompile using gcc. +The verbose flag is added to show what is printed out:: + + >>>inline(code,['a'],compiler='gcc',verbose=2,force=1) + running build_ext + building 'sc_86e98826b65b047ffd2cd5f479c627f13' extension + c:\gcc-2.95.2\bin\g++.exe -mno-cygwin -mdll -O2 -w -Wstrict-prototypes -IC: + \home\ej\wrk\scipy\weave -IC:\Python21\Include -c C:\DOCUME~1\eric\LOCAL + S~1\Temp\python21_compiled\sc_86e98826b65b047ffd2cd5f479c627f13.cpp + -o C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b04ffd2cd5f479c627f13.o + skipping C:\home\ej\wrk\scipy\weave\CXX\cxxextensions.c + (C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxxextensions.o up-to-date) + skipping C:\home\ej\wrk\scipy\weave\CXX\cxxsupport.cxx + (C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxxsupport.o up-to-date) + skipping C:\home\ej\wrk\scipy\weave\CXX\IndirectPythonInterface.cxx + (C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\indirectpythoninterface.o up-to-date) + skipping C:\home\ej\wrk\scipy\weave\CXX\cxx_extensions.cxx + (C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxx_extensions.o + up-to-date) + writing C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ffd2cd5f479c627f13.def + c:\gcc-2.95.2\bin\dllwrap.exe --driver-name g++ -mno-cygwin + -mdll -static --output-lib + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\libsc_86e98826b65b047ffd2cd5f479c627f13.a --def + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ffd2cd5f479c627f13.def + -sC:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\sc_86e98826b65b047ffd2cd5f479c627f13.o + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxxextensions.o + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxxsupport.o + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\indirectpythoninterface.o + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\temp\Release\cxx_extensions.o -LC:\Python21\libs + -lpython21 -o + C:\DOCUME~1\eric\LOCALS~1\Temp\python21_compiled\sc_86e98826b65b047ffd2cd5f479c627f13.pyd + 15 + +That's quite a bit of output. ``verbose=1`` just prints the compile time. + +:: + + >>>inline(code,['a'],compiler='gcc',verbose=1,force=1) + Compiling code... + finished compiling (sec): 6.00800001621 + 15 + + +.. note:: + I've only used the ``compiler`` option for switching between 'msvc' + and 'gcc' on windows. It may have use on Unix also, but I don't know yet. + +The ``support_code`` argument is likely to be used a lot. It allows you to +specify extra code fragments such as function, structure or class definitions +that you want to use in the ``code`` string. Note that changes to +``support_code`` do *not* force a recompile. The catalog only relies on +``code`` (for performance reasons) to determine whether recompiling is +necessary. So, if you make a change to support_code, you'll need to alter +``code`` in some way or use the ``force`` argument to get the code to +recompile. I usually just add some inocuous whitespace to the end of one of +the lines in ``code`` somewhere. Here's an example of defining a separate +method for calculating the string length: + +:: + >>> from weave import inline + >>> a = 'a longer string' + >>> support_code = """ + ... PyObject* length(Py::String a) + ... { + ... int l = a.length(); + ... return Py::new_reference_to(Py::Int(l)); + ... } + ... """ + >>> inline("return_val = length(a);",['a'], + ... support_code = support_code) + 15 + + +``customize`` is a left over from a previous way of specifying compiler +options. It is a ``custom_info`` object that can specify quite a bit of +information about how a file is compiled. These ``info`` objects are the +standard way of defining compile information for type conversion classes. +However, I don't think they are as handy here, especially since we've exposed +all the keyword arguments that distutils can handle. Between these keywords, +and the ``support_code`` option, I think ``customize`` may be obsolete. We'll +see if anyone cares to use it. If not, it'll get axed in the next version. + +The ``type_factories`` variable is important to people who want to customize +the way arguments are converted from Python to C. We'll talk about this in +the next chapter **xx** of this document when we discuss type conversions. + +``auto_downcast`` handles one of the big type conversion issues that is +common when using NumPy arrays in conjunction with Python scalar values. If +you have an array of single precision values and multiply that array by a +Python scalar, the result is upcast to a double precision array because the +scalar value is double precision. This is not usually the desired behavior +because it can double your memory usage. ``auto_downcast`` goes some distance +towards changing the casting precedence of arrays and scalars. If your only +using single precision arrays, it will automatically downcast all scalar +values from double to single precision when they are passed into the C++ +code. This is the default behavior. If you want all values to keep there +default type, set ``auto_downcast`` to 0. + + +Returning Values +---------------- + +Python variables in the local and global scope transfer seemlessly from +Python into the C++ snippets. And, if ``inline`` were to completely live up +to its name, any modifications to variables in the C++ code would be +reflected in the Python variables when control was passed back to Python. For +example, the desired behavior would be something like:: + + # THIS DOES NOT WORK + >>> a = 1 + >>> weave.inline("a++;",['a']) + >>> a + 2 + + +Instead you get:: + + >>> a = 1 + >>> weave.inline("a++;",['a']) + >>> a + 1 + + +Variables are passed into C++ as if you are calling a Python function. +Python's calling convention is sometimes called "pass by assignment". This +means its as if a ``c_a = a`` assignment is made right before ``inline`` call +is made and the ``c_a`` variable is used within the C++ code. Thus, any +changes made to ``c_a`` are not reflected in Python's ``a`` variable. Things +do get a little more confusing, however, when looking at variables with +mutable types. Changes made in C++ to the contents of mutable types *are* +reflected in the Python variables. + +:: + >>> a= [1,2] + >>> weave.inline("PyList_SetItem(a.ptr(),0,PyInt_FromLong(3));",['a']) + >>> print a + [3, 2] + + +So modifications to the contents of mutable types in C++ are seen when +control is returned to Python. Modifications to immutable types such as +tuples, strings, and numbers do not alter the Python variables. If you need +to make changes to an immutable variable, you'll need to assign the new value +to the "magic" variable ``return_val`` in C++. This value is returned by the +``inline()`` function:: + + >>> a = 1 + >>> a = weave.inline("return_val = Py::new_reference_to(Py::Int(a+1));",['a']) + >>> a + 2 + + +The ``return_val`` variable can also be used to return newly created values. +This is possible by returning a tuple. The following trivial example +illustrates how this can be done:: + + # python version + def multi_return(): + return 1, '2nd' + + # C version. + def c_multi_return(): + code = """ + py::tuple results(2); + results[0] = 1; + results[1] = "2nd"; + return_val = results; + """ + return inline_tools.inline(code) + +The example is available in ``examples/tuple_return.py``. It also has the +dubious honor of demonstrating how much ``inline()`` can slow things down. +The C version here is about 7-10 times slower than the Python version. Of +course, something so trivial has no reason to be written in C anyway. + + +The issue with ``locals()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``inline`` passes the ``locals()`` and ``globals()`` dictionaries from Python +into the C++ function from the calling function. It extracts the variables +that are used in the C++ code from these dictionaries, converts then to C++ +variables, and then calculates using them. It seems like it would be trivial, +then, after the calculations were finished to then insert the new values back +into the ``locals()`` and ``globals()`` dictionaries so that the modified +values were reflected in Python. Unfortunately, as pointed out by the Python +manual, the locals() dictionary is not writable. + +I suspect ``locals()`` is not writable because there are some optimizations +done to speed lookups of the local namespace. I'm guessing local lookups +don't always look at a dictionary to find values. Can someone "in the know" +confirm or correct this? Another thing I'd like to know is whether there is a +way to write to the local namespace of another stack frame from C/C++. If so, +it would be possible to have some clean up code in compiled functions that +wrote final values of variables in C++ back to the correct Python stack +frame. I think this goes a long way toward making ``inline`` truely live up +to its name. I don't think we'll get to the point of creating variables in +Python for variables created in C -- although I suppose with a C/C++ parser +you could do that also. + + +A quick look at the code +------------------------ + +``weave`` generates a C++ file holding an extension function for each +``inline`` code snippet. These file names are generated using from the md5 +signature of the code snippet and saved to a location specified by the +PYTHONCOMPILED environment variable (discussed later). The cpp files are +generally about 200-400 lines long and include quite a few functions to +support type conversions, etc. However, the actual compiled function is +pretty simple. Below is the familiar ``printf`` example: + +:: + >>> import weave + >>> a = 1 + >>> weave.inline('printf("%d\\n",a);',['a']) + 1 + + +And here is the extension function generated by ``inline``:: + + static PyObject* compiled_func(PyObject*self, PyObject* args) + { + py::object return_val; + int exception_occured = 0; + PyObject *py__locals = NULL; + PyObject *py__globals = NULL; + PyObject *py_a; + py_a = NULL; + + if(!PyArg_ParseTuple(args,"OO:compiled_func",&py__locals,&py__globals)) + return NULL; + try + { + PyObject* raw_locals = py_to_raw_dict(py__locals,"_locals"); + PyObject* raw_globals = py_to_raw_dict(py__globals,"_globals"); + /* argument conversion code */ + py_a = get_variable("a",raw_locals,raw_globals); + int a = convert_to_int(py_a,"a"); + /* inline code */ + /* NDARRAY API VERSION 90907 */ + printf("%d\n",a); /*I would like to fill in changed locals and globals here...*/ + } + catch(...) + { + return_val = py::object(); + exception_occured = 1; + } + /* cleanup code */ + if(!(PyObject*)return_val && !exception_occured) + { + return_val = Py_None; + } + return return_val.disown(); + } + +Every inline function takes exactly two arguments -- the local and global +dictionaries for the current scope. All variable values are looked up out of +these dictionaries. The lookups, along with all ``inline`` code execution, +are done within a C++ ``try`` block. If the variables aren't found, or there +is an error converting a Python variable to the appropriate type in C++, an +exception is raised. The C++ exception is automatically converted to a Python +exception by SCXX and returned to Python. The ``py_to_int()`` function +illustrates how the conversions and exception handling works. py_to_int first +checks that the given PyObject* pointer is not NULL and is a Python integer. +If all is well, it calls the Python API to convert the value to an ``int``. +Otherwise, it calls ``handle_bad_type()`` which gathers information about +what went wrong and then raises a SCXX TypeError which returns to Python as a +TypeError. + +:: + + int py_to_int(PyObject* py_obj,char* name) + { + if (!py_obj || !PyInt_Check(py_obj)) + handle_bad_type(py_obj,"int", name); + return (int) PyInt_AsLong(py_obj); + } + + +:: + + void handle_bad_type(PyObject* py_obj, char* good_type, char* var_name) + { + char msg[500]; + sprintf(msg,"received '%s' type instead of '%s' for variable '%s'", + find_type(py_obj),good_type,var_name); + throw Py::TypeError(msg); + } + + char* find_type(PyObject* py_obj) + { + if(py_obj == NULL) return "C NULL value"; + if(PyCallable_Check(py_obj)) return "callable"; + if(PyString_Check(py_obj)) return "string"; + if(PyInt_Check(py_obj)) return "int"; + if(PyFloat_Check(py_obj)) return "float"; + if(PyDict_Check(py_obj)) return "dict"; + if(PyList_Check(py_obj)) return "list"; + if(PyTuple_Check(py_obj)) return "tuple"; + if(PyFile_Check(py_obj)) return "file"; + if(PyModule_Check(py_obj)) return "module"; + + //should probably do more interagation (and thinking) on these. + if(PyCallable_Check(py_obj) && PyInstance_Check(py_obj)) return "callable"; + if(PyInstance_Check(py_obj)) return "instance"; + if(PyCallable_Check(py_obj)) return "callable"; + return "unkown type"; + } + +Since the ``inline`` is also executed within the ``try/catch`` block, you can +use CXX exceptions within your code. It is usually a bad idea to directly +``return`` from your code, even if an error occurs. This skips the clean up +section of the extension function. In this simple example, there isn't any +clean up code, but in more complicated examples, there may be some reference +counting that needs to be taken care of here on converted variables. To avoid +this, either uses exceptions or set ``return_val`` to NULL and use +``if/then's`` to skip code after errors. + +Technical Details +================= + +There are several main steps to using C/C++ code withing Python: + +1. Type conversion +2. Generating C/C++ code +3. Compile the code to an extension module +4. Catalog (and cache) the function for future use + +Items 1 and 2 above are related, but most easily discussed separately. Type +conversions are customizable by the user if needed. Understanding them is +pretty important for anything beyond trivial uses of ``inline``. Generating +the C/C++ code is handled by ``ext_function`` and ``ext_module`` classes and +. For the most part, compiling the code is handled by distutils. Some +customizations were needed, but they were relatively minor and do not require +changes to distutils itself. Cataloging is pretty simple in concept, but +surprisingly required the most code to implement (and still likely needs some +work). So, this section covers items 1 and 4 from the list. Item 2 is covered +later in the chapter covering the ``ext_tools`` module, and distutils is +covered by a completely separate document xxx. + + +Passing Variables in/out of the C/C++ code +========================================== + +.. note:: + Passing variables into the C code is pretty straight forward, but + there are subtlties to how variable modifications in C are returned to + Python. see `Returning Values`_ for a more thorough discussion of this issue. + +Type Conversions +================ + +.. note:: + Maybe ``xxx_converter`` instead of ``xxx_specification`` is a more + descriptive name. Might change in future version? + +By default, ``inline()`` makes the following type conversions between Python +and C++ types. + +Default Data Type Conversions + +Python + +C++ + + int int + float double + complex std::complex + string py::string + list py::list + dict py::dict + tuple py::tuple + file FILE* + callable py::object + instance py::object + numpy.ndarray PyArrayObject* + wxXXX wxXXX* + +The ``Py::`` namespace is defined by the SCXX library which has C++ class +equivalents for many Python types. ``std::`` is the namespace of the standard +library in C++. + + +.. note:: + - I haven't figured out how to handle ``long int`` yet (I think they + are currenlty converted to int - - check this). + - Hopefully VTK will be added to the list soon + +Python to C++ conversions fill in code in several locations in the generated +``inline`` extension function. Below is the basic template for the function. +This is actually the exact code that is generated by calling +``weave.inline("")``. + + +The ``/* inline code */`` section is filled with the code passed to the +``inline()`` function call. The ``/*argument convserion code*/`` and ``/* +cleanup code */`` sections are filled with code that handles conversion from +Python to C++ types and code that deallocates memory or manipulates reference +counts before the function returns. The following sections demostrate how +these two areas are filled in by the default conversion methods. * Note: I'm +not sure I have reference counting correct on a few of these. The only thing +I increase/decrease the ref count on is NumPy arrays. If you see an issue, +please let me know. + +NumPy Argument Conversion +------------------------- + +Integer, floating point, and complex arguments are handled in a very similar +fashion. Consider the following inline function that has a single integer +variable passed in:: + + >>> a = 1 + >>> inline("",['a']) + + +The argument conversion code inserted for ``a`` is:: + + /* argument conversion code */ + int a = py_to_int (get_variable("a",raw_locals,raw_globals),"a"); + +``get_variable()`` reads the variable ``a`` from the local and global +namespaces. ``py_to_int()`` has the following form:: + + static int py_to_int(PyObject* py_obj,char* name) + { + if (!py_obj || !PyInt_Check(py_obj)) + handle_bad_type(py_obj,"int", name); + return (int) PyInt_AsLong(py_obj); + } + + +Similarly, the float and complex conversion routines look like:: + + static double py_to_float(PyObject* py_obj,char* name) + { + if (!py_obj || !PyFloat_Check(py_obj)) + handle_bad_type(py_obj,"float", name); + return PyFloat_AsDouble(py_obj); + } + + static std::complex py_to_complex(PyObject* py_obj,char* name) + { + if (!py_obj || !PyComplex_Check(py_obj)) + handle_bad_type(py_obj,"complex", name); + return std::complex(PyComplex_RealAsDouble(py_obj), + PyComplex_ImagAsDouble(py_obj)); + } + +NumPy conversions do not require any clean up code. + +String, List, Tuple, and Dictionary Conversion +---------------------------------------------- + +Strings, Lists, Tuples and Dictionary conversions are all converted to SCXX +types by default. For the following code, + +:: + + >>> a = [1] + >>> inline("",['a']) + + +The argument conversion code inserted for ``a`` is:: + + /* argument conversion code */ + Py::List a = py_to_list(get_variable("a",raw_locals,raw_globals),"a"); + + +``get_variable()`` reads the variable ``a`` from the local and global +namespaces. ``py_to_list()`` and its friends has the following form:: + + static Py::List py_to_list(PyObject* py_obj,char* name) + { + if (!py_obj || !PyList_Check(py_obj)) + handle_bad_type(py_obj,"list", name); + return Py::List(py_obj); + } + + static Py::String py_to_string(PyObject* py_obj,char* name) + { + if (!PyString_Check(py_obj)) + handle_bad_type(py_obj,"string", name); + return Py::String(py_obj); + } + + static Py::Dict py_to_dict(PyObject* py_obj,char* name) + { + if (!py_obj || !PyDict_Check(py_obj)) + handle_bad_type(py_obj,"dict", name); + return Py::Dict(py_obj); + } + + static Py::Tuple py_to_tuple(PyObject* py_obj,char* name) + { + if (!py_obj || !PyTuple_Check(py_obj)) + handle_bad_type(py_obj,"tuple", name); + return Py::Tuple(py_obj); + } + +SCXX handles reference counts on for strings, lists, tuples, and +dictionaries, so clean up code isn't necessary. + +File Conversion +--------------- + +For the following code, + +:: + + >>> a = open("bob",'w') + >>> inline("",['a']) + + +The argument conversion code is:: + + /* argument conversion code */ + PyObject* py_a = get_variable("a",raw_locals,raw_globals); + FILE* a = py_to_file(py_a,"a"); + + +``get_variable()`` reads the variable ``a`` from the local and global +namespaces. ``py_to_file()`` converts PyObject* to a FILE* and increments the +reference count of the PyObject*:: + + FILE* py_to_file(PyObject* py_obj, char* name) + { + if (!py_obj || !PyFile_Check(py_obj)) + handle_bad_type(py_obj,"file", name); + + Py_INCREF(py_obj); + return PyFile_AsFile(py_obj); + } + +Because the PyObject* was incremented, the clean up code needs to decrement +the counter + +:: + + /* cleanup code */ + Py_XDECREF(py_a); + + +Its important to understand that file conversion only works on actual files +-- i.e. ones created using the ``open()`` command in Python. It does not +support converting arbitrary objects that support the file interface into C +``FILE*`` pointers. This can affect many things. For example, in initial +``printf()`` examples, one might be tempted to solve the problem of C and +Python IDE's (PythonWin, PyCrust, etc.) writing to different stdout and +stderr by using ``fprintf()`` and passing in ``sys.stdout`` and +``sys.stderr``. For example, instead of + +:: + + >>> weave.inline('printf("hello\\n");') + + +You might try: + +:: + + >>> buf = sys.stdout + >>> weave.inline('fprintf(buf,"hello\\n");',['buf']) + + +This will work as expected from a standard python interpreter, but in +PythonWin, the following occurs: + +:: + + >>> buf = sys.stdout + >>> weave.inline('fprintf(buf,"hello\\n");',['buf']) + Traceback (most recent call last): + File "", line 1, in ? + File "C:\Python21\weave\inline_tools.py", line 315, in inline + auto_downcast = auto_downcast, + File "C:\Python21\weave\inline_tools.py", line 386, in compile_function + type_factories = type_factories) + File "C:\Python21\weave\ext_tools.py", line 197, in __init__ + auto_downcast, type_factories) + File "C:\Python21\weave\ext_tools.py", line 390, in assign_variable_types + raise TypeError, format_error_msg(errors) + TypeError: {'buf': "Unable to convert variable 'buf' to a C++ type."} + + +The traceback tells us that ``inline()`` was unable to convert 'buf' to a C++ +type (If instance conversion was implemented, the error would have occurred +at runtime instead). Why is this? Let's look at what the ``buf`` object +really is:: + + >>> buf + pywin.framework.interact.InteractiveView instance at 00EAD014 + + +PythonWin has reassigned ``sys.stdout`` to a special object that implements +the Python file interface. This works great in Python, but since the special +object doesn't have a FILE* pointer underlying it, fprintf doesn't know what +to do with it (well this will be the problem when instance conversion is +implemented...). + +Callable, Instance, and Module Conversion +----------------------------------------- + + +.. note:: + Need to look into how ref counts should be handled. Also, Instance and + Module conversion are not currently implemented. + +:: + + >>> def a(): + pass + >>> inline("",['a']) + + +Callable and instance variables are converted to PyObject*. Nothing is done +to there reference counts. + +:: + + /* argument conversion code */ + PyObject* a = py_to_callable(get_variable("a",raw_locals,raw_globals),"a"); + + +``get_variable()`` reads the variable ``a`` from the local and global +namespaces. The ``py_to_callable()`` and ``py_to_instance()`` don't currently +increment the ref count. + +:: + + PyObject* py_to_callable(PyObject* py_obj, char* name) + { + if (!py_obj || !PyCallable_Check(py_obj)) + handle_bad_type(py_obj,"callable", name); + return py_obj; + } + + PyObject* py_to_instance(PyObject* py_obj, char* name) + { + if (!py_obj || !PyFile_Check(py_obj)) + handle_bad_type(py_obj,"instance", name); + return py_obj; + } + +There is no cleanup code for callables, modules, or instances. + +Customizing Conversions +----------------------- + +Converting from Python to C++ types is handled by xxx_specification classes. +A type specification class actually serve in two related but different roles. +The first is in determining whether a Python variable that needs to be +converted should be represented by the given class. The second is as a code +generator that generate C++ code needed to convert from Python to C++ types +for a specific variable. + +When + +:: + + >>> a = 1 + >>> weave.inline('printf("%d",a);',['a']) + + +is called for the first time, the code snippet has to be compiled. In this +process, the variable 'a' is tested against a list of type specifications +(the default list is stored in weave/ext_tools.py). The *first* specification +in the list is used to represent the variable. + +Examples of ``xxx_specification`` are scattered throughout numerous +"xxx_spec.py" files in the ``weave`` package. Closely related to the +``xxx_specification`` classes are ``yyy_info`` classes. These classes contain +compiler, header, and support code information necessary for including a +certain set of capabilities (such as blitz++ or CXX support) in a compiled +module. ``xxx_specification`` classes have one or more ``yyy_info`` classes +associated with them. If you'd like to define your own set of type +specifications, the current best route is to examine some of the existing +spec and info files. Maybe looking over sequence_spec.py and cxx_info.py are +a good place to start. After defining specification classes, you'll need to +pass them into ``inline`` using the ``type_factories`` argument. A lot of +times you may just want to change how a specific variable type is +represented. Say you'd rather have Python strings converted to +``std::string`` or maybe ``char*`` instead of using the CXX string object, +but would like all other type conversions to have default behavior. This +requires that a new specification class that handles strings is written and +then prepended to a list of the default type specifications. Since it is +closer to the front of the list, it effectively overrides the default string +specification. The following code demonstrates how this is done: ... + + +The Catalog +=========== + +``catalog.py`` has a class called ``catalog`` that helps keep track of +previously compiled functions. This prevents ``inline()`` and related +functions from having to compile functions everytime they are called. +Instead, catalog will check an in memory cache to see if the function has +already been loaded into python. If it hasn't, then it starts searching +through persisent catalogs on disk to see if it finds an entry for the given +function. By saving information about compiled functions to disk, it isn't +necessary to re-compile functions everytime you stop and restart the +interpreter. Functions are compiled once and stored for future use. + +When ``inline(cpp_code)`` is called the following things happen: + +1. A fast local cache of functions is checked for the last function + called for ``cpp_code``. If an entry for ``cpp_code`` doesn't exist in + the cache or the cached function call fails (perhaps because the function + doesn't have compatible types) then the next step is to check the + catalog. + +2. The catalog class also keeps an in-memory cache with a list of all + the functions compiled for ``cpp_code``. If ``cpp_code`` has ever been + called, then this cache will be present (loaded from disk). If the cache + isn't present, then it is loaded from disk. + + If the cache is present, each function in the cache is called until + one is found that was compiled for the correct argument types. If none of + the functions work, a new function is compiled with the given argument + types. This function is written to the on-disk catalog as well as into + the in-memory cache. + +3. When a lookup for ``cpp_code`` fails, the catalog looks through the + on-disk function catalogs for the entries. The PYTHONCOMPILED variable + determines where to search for these catalogs and in what order. If + PYTHONCOMPILED is not present several platform dependent locations are + searched. All functions found for ``cpp_code`` in the path are loaded + into the in-memory cache with functions found earlier in the search path + closer to the front of the call list. + + If the function isn't found in the on-disk catalog, then the function + is compiled, written to the first writable directory in the + PYTHONCOMPILED path, and also loaded into the in-memory cache. + + +Function Storage +---------------- + +Function caches are stored as dictionaries where the key is the entire C++ +code string and the value is either a single function (as in the "level 1" +cache) or a list of functions (as in the main catalog cache). On disk +catalogs are stored in the same manor using standard Python shelves. + +Early on, there was a question as to whether md5 check sums of the C++ code +strings should be used instead of the actual code strings. I think this is +the route inline Perl took. Some (admittedly quick) tests of the md5 vs. the +entire string showed that using the entire string was at least a factor of 3 +or 4 faster for Python. I think this is because it is more time consuming to +compute the md5 value than it is to do look-ups of long strings in the +dictionary. Look at the examples/md5_speed.py file for the test run. + + +Catalog search paths and the PYTHONCOMPILED variable +---------------------------------------------------- + +The default location for catalog files on Unix is is ~/.pythonXX_compiled +where XX is version of Python being used. If this directory doesn't exist, it +is created the first time a catalog is used. The directory must be writable. +If, for any reason it isn't, then the catalog attempts to create a directory +based on your user id in the /tmp directory. The directory permissions are +set so that only you have access to the directory. If this fails, I think +you're out of luck. I don't think either of these should ever fail though. On +Windows, a directory called pythonXX_compiled is created in the user's +temporary directory. + +The actual catalog file that lives in this directory is a Python shelve with +a platform specific name such as "nt21compiled_catalog" so that multiple OSes +can share the same file systems without trampling on each other. Along with +the catalog file, the .cpp and .so or .pyd files created by inline will live +in this directory. The catalog file simply contains keys which are the C++ +code strings with values that are lists of functions. The function lists +point at functions within these compiled modules. Each function in the lists +executes the same C++ code string, but compiled for different input +variables. + +You can use the PYTHONCOMPILED environment variable to specify alternative +locations for compiled functions. On Unix this is a colon (':') separated +list of directories. On windows, it is a (';') separated list of directories. +These directories will be searched prior to the default directory for a +compiled function catalog. Also, the first writable directory in the list is +where all new compiled function catalogs, .cpp and .so or .pyd files are +written. Relative directory paths ('.' and '..') should work fine in the +PYTHONCOMPILED variable as should environement variables. + +There is a "special" path variable called MODULE that can be placed in the +PYTHONCOMPILED variable. It specifies that the compiled catalog should reside +in the same directory as the module that called it. This is useful if an +admin wants to build a lot of compiled functions during the build of a +package and then install them in site-packages along with the package. User's +who specify MODULE in their PYTHONCOMPILED variable will have access to these +compiled functions. Note, however, that if they call the function with a set +of argument types that it hasn't previously been built for, the new function +will be stored in their default directory (or some other writable directory +in the PYTHONCOMPILED path) because the user will not have write access to +the site-packages directory. + +An example of using the PYTHONCOMPILED path on bash follows:: + + PYTHONCOMPILED=MODULE:/some/path;export PYTHONCOMPILED; + + +If you are using python21 on linux, and the module bob.py in site-packages +has a compiled function in it, then the catalog search order when calling +that function for the first time in a python session would be:: + + /usr/lib/python21/site-packages/linuxpython_compiled + /some/path/linuxpython_compiled + ~/.python21_compiled/linuxpython_compiled + + +The default location is always included in the search path. + +.. note:: + hmmm. see a possible problem here. I should probably make a sub- + directory such as /usr/lib/python21/site- + packages/python21_compiled/linuxpython_compiled so that library files + compiled with python21 are tried to link with python22 files in some strange + scenarios. Need to check this. + +The in-module cache (in ``weave.inline_tools`` reduces the overhead of +calling inline functions by about a factor of 2. It can be reduced a little +more for type loop calls where the same function is called over and over +again if the cache was a single value instead of a dictionary, but the +benefit is very small (less than 5%) and the utility is quite a bit less. So, +we'll stick with a dictionary as the cache. + + +======= + Blitz +======= + +.. note:: + most of this section is lifted from old documentation. It should be + pretty accurate, but there may be a few discrepancies. + +``weave.blitz()`` compiles NumPy Python expressions for fast execution. For +most applications, compiled expressions should provide a factor of 2-10 +speed-up over NumPy arrays. Using compiled expressions is meant to be as +unobtrusive as possible and works much like pythons exec statement. As an +example, the following code fragment takes a 5 point average of the 512x512 +2d image, b, and stores it in array, a:: + + from scipy import * # or from NumPy import * + a = ones((512,512), Float64) + b = ones((512,512), Float64) + # ...do some stuff to fill in b... + # now average + a[1:-1,1:-1] = (b[1:-1,1:-1] + b[2:,1:-1] + b[:-2,1:-1] \ + + b[1:-1,2:] + b[1:-1,:-2]) / 5. + + +To compile the expression, convert the expression to a string by putting +quotes around it and then use ``weave.blitz``:: + + import weave + expr = "a[1:-1,1:-1] = (b[1:-1,1:-1] + b[2:,1:-1] + b[:-2,1:-1]" \ + "+ b[1:-1,2:] + b[1:-1,:-2]) / 5." + weave.blitz(expr) + + +The first time ``weave.blitz`` is run for a given expression and set of +arguements, C++ code that accomplishes the exact same task as the Python +expression is generated and compiled to an extension module. This can take up +to a couple of minutes depending on the complexity of the function. +Subsequent calls to the function are very fast. Futher, the generated module +is saved between program executions so that the compilation is only done once +for a given expression and associated set of array types. If the given +expression is executed with a new set of array types, the code most be +compiled again. This does not overwrite the previously compiled function -- +both of them are saved and available for exectution. + +The following table compares the run times for standard NumPy code and +compiled code for the 5 point averaging. + +Method Run Time (seconds) +Standard NumPy 0.46349 +blitz (1st time compiling) 78.95526 +blitz (subsequent calls) 0.05843 (factor of 8 speedup) + +These numbers are for a 512x512 double precision image run on a 400 MHz +Celeron processor under RedHat Linux 6.2. + +Because of the slow compile times, its probably most effective to develop +algorithms as you usually do using the capabilities of scipy or the NumPy +module. Once the algorithm is perfected, put quotes around it and execute it +using ``weave.blitz``. This provides the standard rapid prototyping strengths +of Python and results in algorithms that run close to that of hand coded C or +Fortran. + + +Requirements +============ + +Currently, the ``weave.blitz`` has only been tested under Linux with +gcc-2.95-3 and on Windows with Mingw32 (2.95.2). Its compiler requirements +are pretty heavy duty (see the `blitz++ home page`_), so it won't work with +just any compiler. Particularly MSVC++ isn't up to snuff. A number of other +compilers such as KAI++ will also work, but my suspicions are that gcc will +get the most use. + +Limitations +=========== + +1. Currently, ``weave.blitz`` handles all standard mathematic operators + except for the ** power operator. The built-in trigonmetric, log, + floor/ceil, and fabs functions might work (but haven't been tested). It + also handles all types of array indexing supported by the NumPy module. + numarray's NumPy compatible array indexing modes are likewise supported, + but numarray's enhanced (array based) indexing modes are not supported. + + ``weave.blitz`` does not currently support operations that use array + broadcasting, nor have any of the special purpose functions in NumPy such + as take, compress, etc. been implemented. Note that there are no obvious + reasons why most of this functionality cannot be added to scipy.weave, so + it will likely trickle into future versions. Using ``slice()`` objects + directly instead of ``start:stop:step`` is also not supported. + +2. Currently Python only works on expressions that include assignment + such as + + :: + + >>> result = b + c + d + + This means that the result array must exist before calling + ``weave.blitz``. Future versions will allow the following:: + + >>> result = weave.blitz_eval("b + c + d") + +3. ``weave.blitz`` works best when algorithms can be expressed in a + "vectorized" form. Algorithms that have a large number of if/thens and + other conditions are better hand written in C or Fortran. Further, the + restrictions imposed by requiring vectorized expressions sometimes + preclude the use of more efficient data structures or algorithms. For + maximum speed in these cases, hand-coded C or Fortran code is the only + way to go. + +4. ``weave.blitz`` can produce different results than NumPy in certain + situations. It can happen when the array receiving the results of a + calculation is also used during the calculation. The NumPy behavior is to + carry out the entire calculation on the right hand side of an equation + and store it in a temporary array. This temprorary array is assigned to + the array on the left hand side of the equation. blitz, on the other + hand, does a "running" calculation of the array elements assigning values + from the right hand side to the elements on the left hand side + immediately after they are calculated. Here is an example, provided by + Prabhu Ramachandran, where this happens:: + + # 4 point average. + >>> expr = "u[1:-1, 1:-1] = (u[0:-2, 1:-1] + u[2:, 1:-1] + \ + ... "u[1:-1,0:-2] + u[1:-1, 2:])*0.25" + >>> u = zeros((5, 5), 'd'); u[0,:] = 100 + >>> exec (expr) + >>> u + array([[ 100., 100., 100., 100., 100.], + [ 0., 25., 25., 25., 0.], + [ 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0.]]) + + >>> u = zeros((5, 5), 'd'); u[0,:] = 100 + >>> weave.blitz (expr) + >>> u + array([[ 100. , 100. , 100. , 100. , 100. ], + [ 0. , 25. , 31.25 , 32.8125 , 0. ], + [ 0. , 6.25 , 9.375 , 10.546875 , 0. ], + [ 0. , 1.5625 , 2.734375 , 3.3203125, 0. ], + [ 0. , 0. , 0. , 0. , 0. ]]) + + You can prevent this behavior by using a temporary array. + + :: + + >>> u = zeros((5, 5), 'd'); u[0,:] = 100 + >>> temp = zeros((4, 4), 'd'); + >>> expr = "temp = (u[0:-2, 1:-1] + u[2:, 1:-1] + "\ + ... "u[1:-1,0:-2] + u[1:-1, 2:])*0.25;"\ + ... "u[1:-1,1:-1] = temp" + >>> weave.blitz (expr) + >>> u + array([[ 100., 100., 100., 100., 100.], + [ 0., 25., 25., 25., 0.], + [ 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0.], + [ 0., 0., 0., 0., 0.]]) + +5. One other point deserves mention lest people be confused. + ``weave.blitz`` is not a general purpose Python->C compiler. It only + works for expressions that contain NumPy arrays and/or Python scalar + values. This focused scope concentrates effort on the compuationally + intensive regions of the program and sidesteps the difficult issues + associated with a general purpose Python->C compiler. + + +NumPy efficiency issues: What compilation buys you +================================================== + +Some might wonder why compiling NumPy expressions to C++ is beneficial since +operations on NumPy array operations are already executed within C loops. The +problem is that anything other than the simplest expression are executed in +less than optimal fashion. Consider the following NumPy expression:: + + a = 1.2 * b + c * d + + +When NumPy calculates the value for the 2d array, ``a``, it does the +following steps:: + + temp1 = 1.2 * b + temp2 = c * d + a = temp1 + temp2 + + +Two things to note. Since ``c`` is an (perhaps large) array, a large +temporary array must be created to store the results of ``1.2 * b``. The same +is true for ``temp2``. Allocation is slow. The second thing is that we have 3 +loops executing, one to calculate ``temp1``, one for ``temp2`` and one for +adding them up. A C loop for the same problem might look like:: + + for(int i = 0; i < M; i++) + for(int j = 0; j < N; j++) + a[i,j] = 1.2 * b[i,j] + c[i,j] * d[i,j] + + +Here, the 3 loops have been fused into a single loop and there is no longer a +need for a temporary array. This provides a significant speed improvement +over the above example (write me and tell me what you get). + +So, converting NumPy expressions into C/C++ loops that fuse the loops and +eliminate temporary arrays can provide big gains. The goal then,is to convert +NumPy expression to C/C++ loops, compile them in an extension module, and +then call the compiled extension function. The good news is that there is an +obvious correspondence between the NumPy expression above and the C loop. The +bad news is that NumPy is generally much more powerful than this simple +example illustrates and handling all possible indexing possibilities results +in loops that are less than straight forward to write. (take a peak in NumPy +for confirmation). Luckily, there are several available tools that simplify +the process. + + +The Tools +========= + +``weave.blitz`` relies heavily on several remarkable tools. On the Python +side, the main facilitators are Jermey Hylton's parser module and Travis +Oliphant's NumPy module. On the compiled language side, Todd Veldhuizen's +blitz++ array library, written in C++ (shhhh. don't tell David Beazley), does +the heavy lifting. Don't assume that, because it's C++, it's much slower than +C or Fortran. Blitz++ uses a jaw dropping array of template techniques +(metaprogramming, template expression, etc) to convert innocent looking and +readable C++ expressions into to code that usually executes within a few +percentage points of Fortran code for the same problem. This is good. +Unfortunately all the template raz-ma-taz is very expensive to compile, so +the 200 line extension modules often take 2 or more minutes to compile. This +isn't so good. ``weave.blitz`` works to minimize this issue by remembering +where compiled modules live and reusing them instead of re-compiling every +time a program is re-run. + +Parser +------ + +Tearing NumPy expressions apart, examining the pieces, and then rebuilding +them as C++ (blitz) expressions requires a parser of some sort. I can imagine +someone attacking this problem with regular expressions, but it'd likely be +ugly and fragile. Amazingly, Python solves this problem for us. It actually +exposes its parsing engine to the world through the ``parser`` module. The +following fragment creates an Abstract Syntax Tree (AST) object for the +expression and then converts to a (rather unpleasant looking) deeply nested +list representation of the tree. + +:: + + >>> import parser + >>> import scipy.weave.misc + >>> ast = parser.suite("a = b * c + d") + >>> ast_list = ast.tolist() + >>> sym_list = scipy.weave.misc.translate_symbols(ast_list) + >>> pprint.pprint(sym_list) + ['file_input', + ['stmt', + ['simple_stmt', + ['small_stmt', + ['expr_stmt', + ['testlist', + ['test', + ['and_test', + ['not_test', + ['comparison', + ['expr', + ['xor_expr', + ['and_expr', + ['shift_expr', + ['arith_expr', + ['term', + ['factor', ['power', ['atom', ['NAME', 'a']]]]]]]]]]]]]]], + ['EQUAL', '='], + ['testlist', + ['test', + ['and_test', + ['not_test', + ['comparison', + ['expr', + ['xor_expr', + ['and_expr', + ['shift_expr', + ['arith_expr', + ['term', + ['factor', ['power', ['atom', ['NAME', 'b']]]], + ['STAR', '*'], + ['factor', ['power', ['atom', ['NAME', 'c']]]]], + ['PLUS', '+'], + ['term', + ['factor', ['power', ['atom', ['NAME', 'd']]]]]]]]]]]]]]]]], + ['NEWLINE', '']]], + ['ENDMARKER', '']] + + +Despite its looks, with some tools developed by Jermey H., its possible to +search these trees for specific patterns (sub-trees), extract the sub-tree, +manipulate them converting python specific code fragments to blitz code +fragments, and then re-insert it in the parse tree. The parser module +documentation has some details on how to do this. Traversing the new +blitzified tree, writing out the terminal symbols as you go, creates our new +blitz++ expression string. + +Blitz and NumPy +--------------- + +The other nice discovery in the project is that the data structure used for +NumPy arrays and blitz arrays is nearly identical. NumPy stores "strides" as +byte offsets and blitz stores them as element offsets, but other than that, +they are the same. Further, most of the concept and capabilities of the two +libraries are remarkably similar. It is satisfying that two completely +different implementations solved the problem with similar basic +architectures. It is also fortuitous. The work involved in converting NumPy +expressions to blitz expressions was greatly diminished. As an example, +consider the code for slicing an array in Python with a stride:: + + >>> a = b[0:4:2] + c + >>> a + [0,2,4] + + +In Blitz it is as follows:: + + Array<2,int> b(10); + Array<2,int> c(3); + // ... + Array<2,int> a = b(Range(0,3,2)) + c; + + +Here the range object works exactly like Python slice objects with the +exception that the top index (3) is inclusive where as Python's (4) is +exclusive. Other differences include the type declaraions in C++ and +parentheses instead of brackets for indexing arrays. Currently, +``weave.blitz`` handles the inclusive/exclusive issue by subtracting one from +upper indices during the translation. An alternative that is likely more +robust/maintainable in the long run, is to write a PyRange class that behaves +like Python's range. This is likely very easy. + +The stock blitz also doesn't handle negative indices in ranges. The current +implementation of the ``blitz()`` has a partial solution to this problem. It +calculates and index that starts with a '-' sign by subtracting it from the +maximum index in the array so that:: + + upper index limit + /-----\ + b[:-1] -> b(Range(0,Nb[0]-1-1)) + + +This approach fails, however, when the top index is calculated from other +values. In the following scenario, if ``i+j`` evaluates to a negative value, +the compiled code will produce incorrect results and could even core- dump. +Right now, all calculated indices are assumed to be positive. + +:: + + b[:i-j] -> b(Range(0,i+j)) + + +A solution is to calculate all indices up front using if/then to handle the ++/- cases. This is a little work and results in more code, so it hasn't been +done. I'm holding out to see if blitz++ can be modified to handle negative +indexing, but haven't looked into how much effort is involved yet. While it +needs fixin', I don't think there is a ton of code where this is an issue. + +The actual translation of the Python expressions to blitz expressions is +currently a two part process. First, all x:y:z slicing expression are removed +from the AST, converted to slice(x,y,z) and re-inserted into the tree. Any +math needed on these expressions (subtracting from the maximum index, etc.) +are also preformed here. _beg and _end are used as special variables that are +defined as blitz::fromBegin and blitz::toEnd. + +:: + + a[i+j:i+j+1,:] = b[2:3,:] + + +becomes a more verbose:: + + a[slice(i+j,i+j+1),slice(_beg,_end)] = b[slice(2,3),slice(_beg,_end)] + + +The second part does a simple string search/replace to convert to a blitz +expression with the following translations:: + + slice(_beg,_end) -> _all # not strictly needed, but cuts down on code. + slice -> blitz::Range + [ -> ( + ] -> ) + _stp -> 1 + + +``_all`` is defined in the compiled function as ``blitz::Range.all()``. These +translations could of course happen directly in the syntax tree. But the +string replacement is slightly easier. Note that name spaces are maintained +in the C++ code to lessen the likelyhood of name clashes. Currently no effort +is made to detect name clashes. A good rule of thumb is don't use values that +start with '_' or 'py\_' in compiled expressions and you'll be fine. + +Type definitions and coersion +============================= + +So far we've glossed over the dynamic vs. static typing issue between Python +and C++. In Python, the type of value that a variable holds can change +through the course of program execution. C/C++, on the other hand, forces you +to declare the type of value a variables will hold prior at compile time. +``weave.blitz`` handles this issue by examining the types of the variables in +the expression being executed, and compiling a function for those explicit +types. For example:: + + a = ones((5,5),Float32) + b = ones((5,5),Float32) + weave.blitz("a = a + b") + + +When compiling this expression to C++, ``weave.blitz`` sees that the values +for a and b in the local scope have type ``Float32``, or 'float' on a 32 bit +architecture. As a result, it compiles the function using the float type (no +attempt has been made to deal with 64 bit issues). + +What happens if you call a compiled function with array types that are +different than the ones for which it was originally compiled? No biggie, +you'll just have to wait on it to compile a new version for your new types. +This doesn't overwrite the old functions, as they are still accessible. See +the catalog section in the inline() documentation to see how this is handled. +Suffice to say, the mechanism is transparent to the user and behaves like +dynamic typing with the occasional wait for compiling newly typed functions. + +When working with combined scalar/array operations, the type of the array is +*always* used. This is similar to the savespace flag that was recently added +to NumPy. This prevents issues with the following expression perhaps +unexpectedly being calculated at a higher (more expensive) precision that can +occur in Python:: + + >>> a = array((1,2,3),typecode = Float32) + >>> b = a * 2.1 # results in b being a Float64 array. + +In this example, + +:: + + >>> a = ones((5,5),Float32) + >>> b = ones((5,5),Float32) + >>> weave.blitz("b = a * 2.1") + + +the ``2.1`` is cast down to a ``float`` before carrying out the operation. If +you really want to force the calculation to be a ``double``, define ``a`` and +``b`` as ``double`` arrays. + +One other point of note. Currently, you must include both the right hand side +and left hand side (assignment side) of your equation in the compiled +expression. Also, the array being assigned to must be created prior to +calling ``weave.blitz``. I'm pretty sure this is easily changed so that a +compiled_eval expression can be defined, but no effort has been made to +allocate new arrays (and decern their type) on the fly. + + +Cataloging Compiled Functions +============================= + +See `The Catalog`_ section in the ``weave.inline()`` +documentation. + +Checking Array Sizes +==================== + +Surprisingly, one of the big initial problems with compiled code was making +sure all the arrays in an operation were of compatible type. The following +case is trivially easy:: + + a = b + c + + +It only requires that arrays ``a``, ``b``, and ``c`` have the same shape. +However, expressions like:: + + a[i+j:i+j+1,:] = b[2:3,:] + c + + +are not so trivial. Since slicing is involved, the size of the slices, not +the input arrays must be checked. Broadcasting complicates things further +because arrays and slices with different dimensions and shapes may be +compatible for math operations (broadcasting isn't yet supported by +``weave.blitz``). Reductions have a similar effect as their results are +different shapes than their input operand. The binary operators in NumPy +compare the shapes of their two operands just before they operate on them. +This is possible because NumPy treats each operation independently. The +intermediate (temporary) arrays created during sub-operations in an +expression are tested for the correct shape before they are combined by +another operation. Because ``weave.blitz`` fuses all operations into a single +loop, this isn't possible. The shape comparisons must be done and guaranteed +compatible before evaluating the expression. + +The solution chosen converts input arrays to "dummy arrays" that only +represent the dimensions of the arrays, not the data. Binary operations on +dummy arrays check that input array sizes are comptible and return a dummy +array with the size correct size. Evaluating an expression of dummy arrays +traces the changing array sizes through all operations and fails if +incompatible array sizes are ever found. + +The machinery for this is housed in ``weave.size_check``. It basically +involves writing a new class (dummy array) and overloading it math operators +to calculate the new sizes correctly. All the code is in Python and there is +a fair amount of logic (mainly to handle indexing and slicing) so the +operation does impose some overhead. For large arrays (ie. 50x50x50), the +overhead is negligible compared to evaluating the actual expression. For +small arrays (ie. 16x16), the overhead imposed for checking the shapes with +this method can cause the ``weave.blitz`` to be slower than evaluating the +expression in Python. + +What can be done to reduce the overhead? (1) The size checking code could be +moved into C. This would likely remove most of the overhead penalty compared +to NumPy (although there is also some calling overhead), but no effort has +been made to do this. (2) You can also call ``weave.blitz`` with +``check_size=0`` and the size checking isn't done. However, if the sizes +aren't compatible, it can cause a core-dump. So, foregoing size_checking +isn't advisable until your code is well debugged. + + +Creating the Extension Module +============================= + +``weave.blitz`` uses the same machinery as ``weave.inline`` to build the +extension module. The only difference is the code included in the function is +automatically generated from the NumPy array expression instead of supplied +by the user. + +=================== + Extension Modules +=================== + +``weave.inline`` and ``weave.blitz`` are high level tools that generate +extension modules automatically. Under the covers, they use several classes +from ``weave.ext_tools`` to help generate the extension module. The main two +classes are ``ext_module`` and ``ext_function`` (I'd like to add +``ext_class`` and ``ext_method`` also). These classes simplify the process of +generating extension modules by handling most of the "boiler plate" code +automatically. + +.. note:: + ``inline`` actually sub-classes ``weave.ext_tools.ext_function`` to + generate slightly different code than the standard ``ext_function``. + The main difference is that the standard class converts function + arguments to C types, while inline always has two arguments, the + local and global dicts, and the grabs the variables that need to be + convereted to C from these. + +A Simple Example +================ + +The following simple example demonstrates how to build an extension module +within a Python function:: + + # examples/increment_example.py + from weave import ext_tools + + def build_increment_ext(): + """ Build a simple extension with functions that increment numbers. + The extension will be built in the local directory. + """ + mod = ext_tools.ext_module('increment_ext') + + a = 1 # effectively a type declaration for 'a' in the + # following functions. + + ext_code = "return_val = Py::new_reference_to(Py::Int(a+1));" + func = ext_tools.ext_function('increment',ext_code,['a']) + mod.add_function(func) + + ext_code = "return_val = Py::new_reference_to(Py::Int(a+2));" + func = ext_tools.ext_function('increment_by_2',ext_code,['a']) + mod.add_function(func) + + mod.compile() + +The function ``build_increment_ext()`` creates an extension module named +``increment_ext`` and compiles it to a shared library (.so or .pyd) that can +be loaded into Python.. ``increment_ext`` contains two functions, +``increment`` and ``increment_by_2``. The first line of +``build_increment_ext()``, + + mod = ext_tools.ext_module('increment_ext') + + +creates an ``ext_module`` instance that is ready to have ``ext_function`` +instances added to it. ``ext_function`` instances are created much with a +calling convention similar to ``weave.inline()``. The most common call +includes a C/C++ code snippet and a list of the arguments for the function. +The following + + ext_code = "return_val = Py::new_reference_to(Py::Int(a+1));" + func = ext_tools.ext_function('increment',ext_code,['a']) + + +creates a C/C++ extension function that is equivalent to the following Python +function:: + + def increment(a): + return a + 1 + + +A second method is also added to the module and then, + +:: + + mod.compile() + + +is called to build the extension module. By default, the module is created in +the current working directory. This example is available in the +``examples/increment_example.py`` file found in the ``weave`` directory. At +the bottom of the file in the module's "main" program, an attempt to import +``increment_ext`` without building it is made. If this fails (the module +doesn't exist in the PYTHONPATH), the module is built by calling +``build_increment_ext()``. This approach only takes the time consuming ( a +few seconds for this example) process of building the module if it hasn't +been built before. + +:: + + if __name__ == "__main__": + try: + import increment_ext + except ImportError: + build_increment_ext() + import increment_ext + a = 1 + print 'a, a+1:', a, increment_ext.increment(a) + print 'a, a+2:', a, increment_ext.increment_by_2(a) + +.. note:: + If we were willing to always pay the penalty of building the C++ + code for a module, we could store the md5 checksum of the C++ code + along with some information about the compiler, platform, etc. Then, + ``ext_module.compile()`` could try importing the module before it + actually compiles it, check the md5 checksum and other meta-data in + the imported module with the meta-data of the code it just produced + and only compile the code if the module didn't exist or the + meta-data didn't match. This would reduce the above code to:: + + if __name__ == "__main__": + build_increment_ext() + + a = 1 + print 'a, a+1:', a, increment_ext.increment(a) + print 'a, a+2:', a, increment_ext.increment_by_2(a) + +.. note:: + There would always be the overhead of building the C++ code, but it + would only actually compile the code once. You pay a little in overhead and + get cleaner "import" code. Needs some thought. + +If you run ``increment_example.py`` from the command line, you get the +following:: + + [eric at n0]$ python increment_example.py + a, a+1: 1 2 + a, a+2: 1 3 + + +If the module didn't exist before it was run, the module is created. If it +did exist, it is just imported and used. + +Fibonacci Example +================= + +``examples/fibonacci.py`` provides a little more complex example of how to +use ``ext_tools``. Fibonacci numbers are a series of numbers where each +number in the series is the sum of the previous two: 1, 1, 2, 3, 5, 8, etc. +Here, the first two numbers in the series are taken to be 1. One approach to +calculating Fibonacci numbers uses recursive function calls. In Python, it +might be written as:: + + def fib(a): + if a <= 2: + return 1 + else: + return fib(a-2) + fib(a-1) + + +In C, the same function would look something like this:: + + int fib(int a) + { + if(a <= 2) + return 1; + else + return fib(a-2) + fib(a-1); + } + + +Recursion is much faster in C than in Python, so it would be beneficial to +use the C version for fibonacci number calculations instead of the Python +version. We need an extension function that calls this C function to do this. +This is possible by including the above code snippet as "support code" and +then calling it from the extension function. Support code snippets (usually +structure definitions, helper functions and the like) are inserted into the +extension module C/C++ file before the extension function code. Here is how +to build the C version of the fibonacci number generator:: + + def build_fibonacci(): + """ Builds an extension module with fibonacci calculators. + """ + mod = ext_tools.ext_module('fibonacci_ext') + a = 1 # this is effectively a type declaration + + # recursive fibonacci in C + fib_code = """ + int fib1(int a) + { + if(a <= 2) + return 1; + else + return fib1(a-2) + fib1(a-1); + } + """ + ext_code = """ + int val = fib1(a); + return_val = Py::new_reference_to(Py::Int(val)); + """ + fib = ext_tools.ext_function('fib',ext_code,['a']) + fib.customize.add_support_code(fib_code) + mod.add_function(fib) + + mod.compile() + +XXX More about custom_info, and what xxx_info instances are good for. + +.. note:: + recursion is not the fastest way to calculate fibonacci numbers, but + this approach serves nicely for this example. + + +================================================ + Customizing Type Conversions -- Type Factories +================================================ + +not written + +============================= + Things I wish ``weave`` did +============================= + +It is possible to get name clashes if you uses a variable name that is +already defined in a header automatically included (such as ``stdio.h``) For +instance, if you try to pass in a variable named ``stdout``, you'll get a +cryptic error report due to the fact that ``stdio.h`` also defines the name. +``weave`` should probably try and handle this in some way. Other things... + +.. _PyInline: http://pyinline.sourceforge.net/ +.. _SciPy: http://www.scipy.org +.. _mingw32: http://www.mingw.org%3Ewww.mingw.org +.. _NumPy: http://numeric.scipy.org/ +.. _here: http://www.scipy.org/Weave +.. _Python Cookbook: http://aspn.activestate.com/ASPN/Cookbook/Python +.. _binary_search(): + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81188 +.. _website: http://cxx.sourceforge.net/ +.. _This submission: + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52306 +.. _blitz++ home page: http://www.oonumerics.org/blitz/ + +.. + Local Variables: + mode: rst + End: Copied: trunk/scipy/weave/doc/tutorial_original.html (from rev 3449, trunk/scipy/weave/doc/tutorial.html) From scipy-svn at scipy.org Thu Oct 25 15:04:04 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Thu, 25 Oct 2007 14:04:04 -0500 (CDT) Subject: [Scipy-svn] r3463 - in trunk/scipy/sandbox/maskedarray: . tests Message-ID: <20071025190404.DEAB239C09C@new.scipy.org> Author: pierregm Date: 2007-10-25 14:03:56 -0500 (Thu, 25 Oct 2007) New Revision: 3463 Modified: trunk/scipy/sandbox/maskedarray/core.py trunk/scipy/sandbox/maskedarray/tests/test_core.py trunk/scipy/sandbox/maskedarray/tests/test_extras.py Log: core: fixed imag and real Modified: trunk/scipy/sandbox/maskedarray/core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/core.py 2007-10-25 17:09:53 UTC (rev 3462) +++ trunk/scipy/sandbox/maskedarray/core.py 2007-10-25 19:03:56 UTC (rev 3463) @@ -1528,6 +1528,20 @@ raise MAError, 'Cannot convert masked element to a Python int.' return int(self.item()) #............................................ + def get_imag(self): + result = self._data.imag.view(type(self)) + result.__setmask__(self._mask) + return result + imag = property(fget=get_imag,doc="Imaginary part") + + def get_real(self): + result = self._data.real.view(type(self)) + result.__setmask__(self._mask) + return result + real = property(fget=get_real,doc="Real part") + + + #............................................ def count(self, axis=None): """Counts the non-masked elements of the array along the given axis. @@ -2955,4 +2969,11 @@ assert_equal(mxbig.all(1), [False, False, True]) assert_equal(mxbig.any(0),[False, False, True]) assert_equal(mxbig.any(1), [True, True, True]) + + if 1: + xx = array([1+10j,20+2j], mask=[1,0]) + assert_equal(xx.imag,[10,2]) + assert_equal(xx.imag.filled(), [1e+20,2]) + assert_equal(xx.real,[1,20]) + assert_equal(xx.real.filled(), [1e+20,20]) \ No newline at end of file Modified: trunk/scipy/sandbox/maskedarray/tests/test_core.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_core.py 2007-10-25 17:09:53 UTC (rev 3462) +++ trunk/scipy/sandbox/maskedarray/tests/test_core.py 2007-10-25 19:03:56 UTC (rev 3463) @@ -783,6 +783,15 @@ data_fixed = fix_invalid(data) assert_equal(data_fixed._data, [data.fill_value, 0., 1.]) assert_equal(data_fixed._mask, [1., 0., 1.]) + # + def check_imag_real(self): + xx = array([1+10j,20+2j], mask=[1,0]) + assert_equal(xx.imag,[10,2]) + assert_equal(xx.imag.filled(), [1e+20,2]) + assert_equal(xx.imag.dtype, xx._data.imag.dtype) + assert_equal(xx.real,[1,20]) + assert_equal(xx.real.filled(), [1e+20,20]) + assert_equal(xx.real.dtype, xx._data.real.dtype) #............................................................................... Modified: trunk/scipy/sandbox/maskedarray/tests/test_extras.py =================================================================== --- trunk/scipy/sandbox/maskedarray/tests/test_extras.py 2007-10-25 17:09:53 UTC (rev 3462) +++ trunk/scipy/sandbox/maskedarray/tests/test_extras.py 2007-10-25 19:03:56 UTC (rev 3463) @@ -330,4 +330,4 @@ if __name__ == "__main__": NumpyTest().run() - + \ No newline at end of file From scipy-svn at scipy.org Fri Oct 26 05:39:47 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 26 Oct 2007 04:39:47 -0500 (CDT) Subject: [Scipy-svn] r3464 - trunk/scipy/optimize Message-ID: <20071026093947.61A6339C1BD@new.scipy.org> Author: dmitrey.kroshko Date: 2007-10-26 04:39:42 -0500 (Fri, 26 Oct 2007) New Revision: 3464 Modified: trunk/scipy/optimize/optimize.py Log: information in docstring about using scipy.optimize.fmin_ncg via scikits.openopt Modified: trunk/scipy/optimize/optimize.py =================================================================== --- trunk/scipy/optimize/optimize.py 2007-10-25 19:03:56 UTC (rev 3463) +++ trunk/scipy/optimize/optimize.py 2007-10-26 09:39:42 UTC (rev 3464) @@ -1012,8 +1012,8 @@ If True, return a list of results at each iteration. *Notes* - - Only one of `fhess_p` or `fhess` need to be given. If `fhess` + 1. scikits.openopt offers a unified syntax to call this and other solvers. + 2. Only one of `fhess_p` or `fhess` need to be given. If `fhess` is provided, then `fhess_p` will be ignored. If neither `fhess` nor `fhess_p` is provided, then the hessian product will be approximated using finite differences on `fprime`. `fhess_p` From scipy-svn at scipy.org Fri Oct 26 19:26:31 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Fri, 26 Oct 2007 18:26:31 -0500 (CDT) Subject: [Scipy-svn] r3466 - in trunk/scipy/sandbox/multigrid: . tests Message-ID: <20071026232631.C8A1839C030@new.scipy.org> Author: wnbell Date: 2007-10-26 18:26:11 -0500 (Fri, 26 Oct 2007) New Revision: 3466 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/sa.py trunk/scipy/sandbox/multigrid/tests/test_sa.py Log: aSA now supports blocks Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-26 19:05:34 UTC (rev 3465) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-26 23:26:11 UTC (rev 3466) @@ -12,6 +12,8 @@ def augment_candidates(AggOp, old_Q, old_R, new_candidate): + #TODO update P also + K = old_R.shape[1] #determine blocksizes @@ -36,14 +38,11 @@ Q_data[:,:old_bs[0],:old_bs[1]] = old_Q.data.reshape((-1,) + old_bs) #TODO BSR change # coarse candidates - #R = zeros(((K+1)*AggOp.shape[1],K+1)) - #for i in range(K): - # R[i::K+1,:K] = old_R[i::K,:] R = zeros((AggOp.shape[1],K+1,K+1)) R[:,:K,:K] = old_R.reshape(-1,K,K) c = new_candidate.reshape(-1)[diff(AggOp.indptr) == 1] #eliminate DOFs that aggregation misses - threshold = 1e-10 / abs(c).max() # cutoff for small basis functions + threshold = 1e-10 * abs(c).max() # cutoff for small basis functions X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) @@ -65,7 +64,7 @@ col_norms = 1.0/col_norms col_norms[mask] = 0 X.data *= col_norms[X.indices] - Q_data[:,:,-1] = X.data.reshape(-1,1) + Q_data[:,:,-1] = X.data.reshape(-1,new_bs[0]) Q_data = Q_data.reshape(-1) #TODO BSR change R = R.reshape(-1,K+1) @@ -77,45 +76,8 @@ -def orthonormalize_prolongator(P_l,x_l,W_l,W_m): - """ - - """ - #candidate prolongator (assumes every value from x is used) #TODO permit gaps - X = csr_matrix((x_l,W_l.indices,W_l.indptr),dims=W_l.shape,check=False) - - R = (P_l.T.tocsr() * X) # R has at most 1 nz per row - X = X - P_l*R # othogonalize X against P_l - - #TODO DROP REDUNDANT COLUMNS FROM P (AND R?) HERE (NULL OUT R ACCORDINGLY?) - #TODO REMOVE CORRESPONDING COLUMNS FROM W_l AND ROWS FROM A_m ALSO - W_l_new = W_l - W_m_new = W_m - #normalize surviving columns of X - col_norms = ravel(sqrt(csr_matrix((X.data*X.data,X.indices,X.indptr),dims=X.shape,check=False).sum(axis=0))) - print "zero cols",sum(col_norms == 0) - print "small cols",sum(col_norms < 1e-8) - Xcopy = X.copy() - X.data *= (1.0/col_norms)[X.indices] - - P_l_new = hstack_csr(P_l,X) - - - #check orthonormality - print "norm(P.T*P - I) ",scipy.linalg.norm((P_l_new.T * P_l_new - scipy.sparse.spidentity(P_l_new.shape[1])).data) - #assert(scipy.linalg.norm((P_l_new.T * P_l_new - scipy.sparse.spidentity(P_l_new.shape[1])).data)<1e-8) - - x_m = zeros(P_l_new.shape[1],dtype=x_l.dtype) - x_m[:P_l.shape[1]][diff(R.indptr).astype('bool')] = R.data - x_m[P_l.shape[1]:] = col_norms - - print "||x_l - P_l*x_m||",scipy.linalg.norm(P_l_new* x_m - x_l) #see if x_l is represented exactly - - return P_l_new,x_m,W_l,W_m - - def smoothed_prolongator(P,A): #just use Richardson for now #omega = 4.0/(3.0*approximate_spectral_radius(A)) @@ -132,7 +94,7 @@ -def sa_hierarchy(A,Ws,B): +def sa_hierarchy(A,B,AggOps): """ Construct multilevel hierarchy using Smoothed Aggregation Inputs: @@ -150,8 +112,8 @@ Ts = [] Bs = [B] - for W in Ws: - P,B = sa_fit_candidates(W,B) + for AggOp in AggOps: + P,B = sa_fit_candidates(AggOp,B) I = smoothed_prolongator(P,A) A = I.T.tocsr() * A * I As.append(A) @@ -160,14 +122,10 @@ Bs.append(B) return As,Ps,Ts,Bs -def make_bridge(I,N): - tail = I.indptr[-1].repeat(N - I.shape[0]) - ptr = concatenate((I.indptr,tail)) - return csr_matrix((I.data,I.indices,ptr),dims=(N,I.shape[1]),check=False) class adaptive_sa_solver: - def __init__(self,A,blocks=None,options=None,max_levels=10,max_coarse=100,\ - max_candidates=1,mu=5,epsilon=0.1): + def __init__(self, A, blocks=None, max_levels=10, max_coarse=100,\ + max_candidates=1, mu=5, epsilon=0.1): self.A = A @@ -181,29 +139,22 @@ max_levels = max_levels, \ max_coarse = max_coarse, \ mu = mu, epsilon = epsilon) - - #Ws = AggOps - x /= abs(x).max() - fine_candidates = x - + #create SA using x here - As,Ps,Ts,self.candidates = sa_hierarchy(A,AggOps,fine_candidates) - #self.candidates = [x] + As,Ps,Ts,Bs = sa_hierarchy(A,x,AggOps) for i in range(max_candidates - 1): - #x = self.__develop_candidate_OLD(A,As,Ps,Ts,Ws,AggOps,mu=mu) - #As,Ps,Ts,Ws = self.__augment_cycle(A,As,Ts,Ws,AggOps,x) - #self.candidates.append(x) + x = self.__develop_new_candidate(As,Ps,Ts,Bs,AggOps,mu=mu) #TODO which is faster? - x = self.__develop_candidate(As,Ps,Ts,AggOps,self.candidates,mu=mu) - x /= abs(x).max() - fine_candidates = hstack((fine_candidates,x)) - As,Ps,Ts,self.candidates = sa_hierarchy(A,AggOps,fine_candidates) + #As,Ps,Ts,Bs = self.__augment_cycle(As,Ps,Ts,Bs,AggOps,x) + B = hstack((Bs[0],x)) + As,Ps,Ts,Bs = sa_hierarchy(A,B,AggOps) self.Ts = Ts self.solver = multilevel_solver(As,Ps) self.AggOps = AggOps + self.Bs = Bs def __initialization_stage(self,A,blocks,max_levels,max_coarse,mu,epsilon): @@ -232,6 +183,8 @@ P_l,x = sa_fit_candidates(W_l,x) #step 4c I_l = smoothed_prolongator(P_l,A_l) #step 4d A_l = I_l.T.tocsr() * A_l * I_l #step 4e + + blocks = None #not needed on subsequent levels print 'A_l.shape',A_l.shape AggOps.append(W_l) @@ -260,7 +213,7 @@ return x,AggOps #first candidate,aggregation - def __develop_candidate(self,As,Ps,Ts,AggOps,candidates,mu): + def __develop_new_candidate(self,As,Ps,Ts,Bs,AggOps,mu): A = As[0] x = randn(A.shape[0],1) @@ -281,22 +234,7 @@ return csr_matrix((P.data,P.indices,indptr),dims=(K*P.shape[0]/(K-1),P.shape[1])) for i in range(len(As) - 2): - #TODO test augment_candidates against fit candidates - T,R = augment_candidates(AggOps[i], Ts[i], candidates[i+1], x) - #if i == 0: - # temp = hstack((candidates[0],x)) - #else: - # K = candidates[i].shape[1] - # temp = zeros((x.shape[0]/(K+1),K+1,K + 1)) - # temp[:,:-1,:-1] = candidates[i].reshape(-1,K,K) - # temp[:,:,-1] = x.reshape(-1,K+1,1) - # temp = temp.reshape(-1,K+1) - #T_,R_ = sa_fit_candidates(AggOps[i],temp) - #print "T - T_",abs(T - T_).sum() - #assert(abs(T - T_).sum() < 1e-10) - #assert((R - R_).sum() < 1e-10) - #T,R = T_,R_ - #TODO end test + T,R = augment_candidates(AggOps[i], Ts[i], Bs[i+1], x) P = smoothed_prolongator(T,A) A = P.T.tocsr() * A * P @@ -316,87 +254,32 @@ x = P * x gauss_seidel(A,x,zeros_like(x),iterations=mu,sweep='symmetric') - #for P in reversed(temp_Ps): - # x = P*x - return x - -## def __develop_candidate_OLD(self,A,As,Ps,Ts,Ws,AggOps,mu): -## #scipy.random.seed(0) #TEST -## x = scipy.rand(A.shape[0]) -## b = zeros_like(x) -## -## solver = multilevel_solver(As,Ps) -## -## x = solver.solve(b, x0=x, tol=1e-10, maxiter=mu) -## -## #TEST FOR CONVERGENCE HERE -## -## A_l,P_l,W_l,x_l = As[0],Ts[0],Ws[0],x -## -## temp_Ps = [] -## for i in range(len(As) - 2): -## P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, AggOps[i+1]) -## -## I_l_new = smoothed_prolongator(P_l_new,A_l) -## A_m_new = I_l_new.T.tocsr() * A_l * I_l_new -## bridge = make_bridge(Ps[i+1],A_m_new.shape[0]) -## -## temp_solver = multilevel_solver( [A_m_new] + As[i+2:], [bridge] + Ps[i+2:] ) -## -## for n in range(mu): -## x_m = temp_solver.solve(zeros_like(x_m), x0=x_m, tol=1e-8, maxiter=1) -## -## temp_Ps.append(I_l_new) -## -## W_l = vstack_csr(Ws[i+1],W_m_new) #prepare for next iteration -## A_l = A_m_new -## x_l = x_m -## P_l = make_bridge(Ts[i+1],A_m_new.shape[0]) -## -## x = x_l -## for I in reversed(temp_Ps): -## x = I*x -## -## return x - - - def __augment_cycle(self,A,As,Ts,Ws,AggOps,x): - #make a new cycle using the new candidate - A_l,P_l,W_l,x_l = As[0],Ts[0],AggOps[0],x + def __augment_cycle(self,As,Ps,Ts,Bs,AggOps,x): + A = As[0] - new_As,new_Ps,new_Ts,new_Ws = [A],[],[],[AggOps[0]] + new_As = [A] + new_Ps = [] + new_Ts = [] + new_Bs = [ hstack((Bs[0],x)) ] - for i in range(len(As) - 2): - P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, AggOps[i+1]) + for i in range(len(As) - 1): + T,R = augment_candidates(AggOps[i], Ts[i], Bs[i+1], x) - I_l_new = smoothed_prolongator(P_l_new,A_l) - A_m_new = I_l_new.T.tocsr() * A_l * I_l_new - W_m_new = vstack_csr(Ws[i+1],W_m_new) + P = smoothed_prolongator(T,A) + A = P.T.tocsr() * A * P - new_As.append(A_m_new) - new_Ws.append(W_m_new) - new_Ps.append(I_l_new) - new_Ts.append(P_l_new) + new_As.append(A) + new_Ps.append(P) + new_Ts.append(T) + new_Bs.append(R) - #prepare for next iteration - W_l = W_m_new - A_l = A_m_new - x_l = x_m - P_l = make_bridge(Ts[i+1],A_m_new.shape[0]) + x = R[:,-1].reshape(-1,1) - P_l_new, x_m, W_l_new, W_m_new = orthonormalize_prolongator(P_l, x_l, W_l, csr_matrix((P_l.shape[1],1))) - I_l_new = smoothed_prolongator(P_l_new,A_l) - A_m_new = I_l_new.T.tocsr() * A_l * I_l_new + return new_As,new_Ps,new_Ts,new_Bs - new_As.append(A_m_new) - new_Ps.append(I_l_new) - new_Ts.append(P_l_new) - return new_As,new_Ps,new_Ts,new_Ws - - if __name__ == '__main__': from scipy import * from utils import diag_sparse @@ -416,8 +299,11 @@ A = io.mmread("tests/sample_data/elas30_A.mtx").tocsr() blocks = arange(A.shape[0]/2).repeat(2) - - asa = adaptive_sa_solver(A,max_candidates=3,mu=10) + + from time import clock; start = clock() + asa = adaptive_sa_solver(A,max_candidates=5,mu=6,blocks=blocks) + print "Adaptive Solver Construction: %s seconds" % (clock() - start); del start + #scipy.random.seed(0) #make tests repeatable x = randn(A.shape[0]) b = A*randn(A.shape[0]) @@ -445,7 +331,6 @@ print "constant Rayleigh quotient",dot(ones(A.shape[0]),A*ones(A.shape[0]))/float(A.shape[0]) - def plot2d_arrows(x): from pylab import figure,quiver,show x = x.reshape(-1) @@ -467,7 +352,7 @@ pcolor(x.reshape(sqrt(len(x)),sqrt(len(x)))) show() - for c in asa.candidates[0].T: + for c in asa.Bs[0].T: plot2d_arrows(c) print "candidate Rayleigh quotient",dot(c,A*c)/dot(c,c) Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-26 19:05:34 UTC (rev 3465) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-10-26 23:26:11 UTC (rev 3466) @@ -101,8 +101,9 @@ return csr_matrix((Px,Pj,Pp)) -def sa_fit_candidates(AggOp,candidates): - +def sa_fit_candidates(AggOp,candidates,tol=1e-10): + #TODO handle non-floating point candidates better + K = candidates.shape[1] # num candidates N_fine,N_coarse = AggOp.shape @@ -115,12 +116,12 @@ R = zeros((N_coarse,K,K)) #storage for coarse candidates candidate_matrices = [] - + for i in range(K): c = candidates[:,i] c = c[diff(AggOp.indptr) == 1] # eliminate DOFs that aggregation misses - threshold = 1e-10 / abs(c).max() # cutoff for small basis functions + threshold = tol * abs(c).max() # cutoff for small basis functions X = csr_matrix((c,AggOp.indices,AggOp.indptr),dims=AggOp.shape) Modified: trunk/scipy/sandbox/multigrid/tests/test_sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-26 19:05:34 UTC (rev 3465) +++ trunk/scipy/sandbox/multigrid/tests/test_sa.py 2007-10-26 23:26:11 UTC (rev 3466) @@ -129,25 +129,25 @@ ## tests where AggOp includes all DOFs #one candidate - self.cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)), ones((5,1)) )) - self.cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)), ones((5,1)) )) - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), ones((9,1)) )) + #self.cases.append((csr_matrix((ones(5),array([0,0,0,1,1]),arange(6)),dims=(5,2)), ones((5,1)) )) + #self.cases.append((csr_matrix((ones(5),array([1,1,0,0,0]),arange(6)),dims=(5,2)), ones((5,1)) )) + #self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), ones((9,1)) )) self.cases.append((csr_matrix((ones(9),array([2,1,0,0,1,2,1,0,2]),arange(10)),dims=(9,3)), arange(9).reshape(9,1) )) #two candidates - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)), vstack((ones(4),arange(4))).T )) - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((ones(9),arange(9))).T )) - self.cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)), vstack((ones(9),arange(9))).T )) - #block candidates - self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((array([1]*9 + [0]*9),arange(2*9))).T )) - - ## tests where AggOp excludes some DOFs - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), ones((5,1)) )) - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5))).T )) + #self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),arange(5)),dims=(4,2)), vstack((ones(4),arange(4))).T )) + #self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((ones(9),arange(9))).T )) + #self.cases.append((csr_matrix((ones(9),array([0,0,1,1,2,2,3,3,3]),arange(10)),dims=(9,4)), vstack((ones(9),arange(9))).T )) + ##block candidates + #self.cases.append((csr_matrix((ones(9),array([0,0,0,1,1,1,2,2,2]),arange(10)),dims=(9,3)), vstack((array([1]*9 + [0]*9),arange(2*9))).T )) + # + ### tests where AggOp excludes some DOFs + #self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), ones((5,1)) )) + #self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5))).T )) - # overdetermined blocks - self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5),arange(5)**2)).T )) - self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9),arange(9)**2)).T )) - self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9))).T )) + ## overdetermined blocks + #self.cases.append((csr_matrix((ones(4),array([0,0,1,1]),array([0,1,2,2,3,4])),dims=(5,2)), vstack((ones(5),arange(5),arange(5)**2)).T )) + #self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9),arange(9)**2)).T )) + #self.cases.append((csr_matrix((ones(6),array([1,3,0,2,1,0]),array([0,0,1,2,2,3,4,5,5,6])),dims=(9,4)), vstack((ones(9),arange(9))).T )) def check_all_cases(self): """Test case where aggregation includes all fine nodes""" From scipy-svn at scipy.org Sat Oct 27 13:09:11 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sat, 27 Oct 2007 12:09:11 -0500 (CDT) Subject: [Scipy-svn] r3467 - trunk/scipy/io Message-ID: <20071027170911.B937339C0E6@new.scipy.org> Author: wnbell Date: 2007-10-27 12:09:07 -0500 (Sat, 27 Oct 2007) New Revision: 3467 Modified: trunk/scipy/io/mio4.py Log: cast index arrays to integer type Modified: trunk/scipy/io/mio4.py =================================================================== --- trunk/scipy/io/mio4.py 2007-10-26 23:26:11 UTC (rev 3466) +++ trunk/scipy/io/mio4.py 2007-10-27 17:09:07 UTC (rev 3467) @@ -166,7 +166,7 @@ res = self.read_array() tmp = res[:-1,:] dims = res[-1,0:2] - ij = N.transpose(tmp[:,0:2]) - 1 # for 1-based indexing + ij = N.transpose(tmp[:,0:2]).astype('i') - 1 # for 1-based indexing vals = tmp[:,2] if res.shape[1] == 4: vals = vals + res[:-1,3] * 1j From scipy-svn at scipy.org Sat Oct 27 14:25:14 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sat, 27 Oct 2007 13:25:14 -0500 (CDT) Subject: [Scipy-svn] r3468 - trunk/scipy/sparse Message-ID: <20071027182514.A546E39C10F@new.scipy.org> Author: wnbell Date: 2007-10-27 13:25:05 -0500 (Sat, 27 Oct 2007) New Revision: 3468 Modified: trunk/scipy/sparse/sparse.py Log: for non-integer dtypes, replace exception with warning and cast Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-27 17:09:07 UTC (rev 3467) +++ trunk/scipy/sparse/sparse.py 2007-10-27 18:25:05 UTC (rev 3468) @@ -514,10 +514,14 @@ class _cs_matrix(spmatrix): def _check(self): - if self.indptr.dtype > numpy.dtype('int64'): - raise TypeError,'indptr array has invalid dtype' - if self.indices.dtype > numpy.dtype('int64'): - raise TypeError,'indices array has invalid dtype' + if self.indptr.dtype.kind != 'i': + warnings.warn("indptr array has non-integer dtype. " \ + "Casting from %s to int" % self.indptr.dtype.name ) + self.indptr = self.indptr.astype('i') + if self.indices.dtype.kind != 'i': + warnings.warn("indices array has non-integer dtype. " \ + "Casting from %s to int" % self.indices.dtype.name ) + self.indices = self.indices.astype('i') #TODO handle non-native byteorder better self.indptr = to_native(self.indptr) From scipy-svn at scipy.org Sat Oct 27 16:44:06 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sat, 27 Oct 2007 15:44:06 -0500 (CDT) Subject: [Scipy-svn] r3469 - in trunk/scipy/sandbox/timeseries: . lib/tests tests Message-ID: <20071027204406.EFA9339C05F@new.scipy.org> Author: pierregm Date: 2007-10-27 15:44:00 -0500 (Sat, 27 Oct 2007) New Revision: 3469 Modified: trunk/scipy/sandbox/timeseries/extras.py trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py trunk/scipy/sandbox/timeseries/tests/test_timeseries.py trunk/scipy/sandbox/timeseries/tests/test_trecords.py trunk/scipy/sandbox/timeseries/trecords.py trunk/scipy/sandbox/timeseries/tseries.py Log: tseries : introduce concatenate and deprecate concatenate_series : fill_missing_dates : fixed fill_value inheritance trecords : fixed printing of record extras : count_missing : added support for quarterly periods. Modified: trunk/scipy/sandbox/timeseries/extras.py =================================================================== --- trunk/scipy/sandbox/timeseries/extras.py 2007-10-27 18:25:05 UTC (rev 3468) +++ trunk/scipy/sandbox/timeseries/extras.py 2007-10-27 20:44:00 UTC (rev 3469) @@ -69,6 +69,22 @@ isfeb = (months == 2) missing[isfeb] -= 2 missing[isfeb & ~isleapyear(series.year)] -= 1 + elif period == 92 and (freq//_c.FR_QTR == 1): + # row: quarters, cold:days + months = series.months + if freq in (_c.FR_QTREJAN, _c.FR_QTRSJAN, _c.FR_QTREAPR, _c.FR_QTRSAPR, + _c.FR_QTREOCT, _c.FR_QTRSOCT, _c.FR_QTREOCT, _c.FR_QTRSOCT): + isfeb = (months == 4) + missing[isfeb] -= 2 + elif freq in (_c.FR_QTREFEB, _c.FR_QTRSFEB, _c.FR_QTREMAY, _c.FR_QTRSMAY, + _c.FR_QTREAUG, _c.FR_QTRSAUG, _c.FR_QTRENOV, _c.FR_QTRSNOV): + missing[numpy.array([m in [2,11] for m in months])] -= 1 + isfeb = (months == 2) + elif freq in (_c.FR_QTREMAR, _c.FR_QTRSMAR, _c.FR_QTREJUN, _c.FR_QTRSJUN, + _c.FR_QTRESEP, _c.FR_QTRSSEP, _c.FR_QTREDEC, _c.FR_QTRSDEC): + missing[numpy.array([m in [3,6] for m in months])] -= 1 + isfeb = (months == 3) + missing[isfeb & ~isleapyear(series.year)] -= 1 elif period not in (12,7): raise NotImplementedError, "Not yet implemented for that frequency..." return missing Modified: trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py =================================================================== --- trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py 2007-10-27 18:25:05 UTC (rev 3468) +++ trunk/scipy/sandbox/timeseries/lib/tests/test_moving_funcs.py 2007-10-27 20:44:00 UTC (rev 3469) @@ -143,4 +143,4 @@ #------------------------------------------------------------------------------ if __name__ == "__main__": - NumpyTest().run() + NumpyTest().run() \ No newline at end of file Modified: trunk/scipy/sandbox/timeseries/tests/test_timeseries.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_timeseries.py 2007-10-27 18:25:05 UTC (rev 3468) +++ trunk/scipy/sandbox/timeseries/tests/test_timeseries.py 2007-10-27 20:44:00 UTC (rev 3469) @@ -29,7 +29,7 @@ from timeseries import Date, date_array_fromlist, date_array_fromrange, date_array, thisday from timeseries import time_series, TimeSeries, adjust_endpoints, \ mask_period, align_series, align_with, fill_missing_dates, tsmasked, \ - concatenate_series, stack, split + concatenate, stack, split class TestCreation(NumpyTestCase): "Base test class for MaskedArrays." @@ -601,27 +601,30 @@ def test_concatenate(self): "Tests concatenate" dlist = ['2007-%02i' % i for i in range(1,6)] - dates = date_array_fromlist(dlist) - data = masked_array(numeric.arange(5), mask=[1,0,0,0,0], dtype=float_) + _dates = date_array_fromlist(dlist) + data = masked_array(numpy.arange(5), mask=[1,0,0,0,0], dtype=float_) # - ser_1 = time_series(data, dates) - ser_2 = time_series(data, dates=dates+10) - newseries = concatenate_series(ser_1, ser_2) - assert_equal(newseries._data,[0,1,2,3,4,0,0,0,0,0,0,1,2,3,4]) + ser_1 = time_series(data, _dates) + ser_2 = time_series(data, dates=_dates+10) + newseries = concatenate((ser_1, ser_2), fill_missing=True) + assert_equal(newseries._series,[0,1,2,3,4,0,0,0,0,0,0,1,2,3,4]) assert_equal(newseries._mask,[1,0,0,0,0]+[1]*5+[1,0,0,0,0]) + assert ~(newseries.has_missing_dates()) # - ser_1 = time_series(data, dates) - ser_2 = time_series(data, dates=dates+10) - newseries = concatenate_series(ser_1, ser_2, keep_gap=False) + ser_1 = time_series(data, _dates) + ser_2 = time_series(data, dates=_dates+10) + newseries = concatenate((ser_1, ser_2), keep_gap=False) assert_equal(newseries._data,[0,1,2,3,4,0,1,2,3,4]) assert_equal(newseries._mask,[1,0,0,0,0]+[1,0,0,0,0]) assert newseries.has_missing_dates() # - ser_2 = time_series(data, dates=dates+3) - newseries = concatenate_series(ser_1, ser_2) - assert_equal(newseries._data,[0,1,2,0,1,2,3,4]) - assert_equal(newseries._mask,[1,0,0,1,0,0,0,0]) + ser_2 = time_series(data, dates=_dates+3) + newseries = concatenate((ser_1, ser_2)) + assert_equal(newseries._data,[0,1,2,3,4,2,3,4]) + assert_equal(newseries._mask,[1,0,0,0,0,0,0,0]) # + newseries = concatenate((ser_1, ser_1[::-1])) + assert_equal(newseries, ser_1) Modified: trunk/scipy/sandbox/timeseries/tests/test_trecords.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_trecords.py 2007-10-27 18:25:05 UTC (rev 3468) +++ trunk/scipy/sandbox/timeseries/tests/test_trecords.py 2007-10-27 20:44:00 UTC (rev 3469) @@ -186,4 +186,4 @@ ############################################################################### #------------------------------------------------------------------------------ if __name__ == "__main__": - NumpyTest().run() + NumpyTest().run() \ No newline at end of file Modified: trunk/scipy/sandbox/timeseries/trecords.py =================================================================== --- trunk/scipy/sandbox/timeseries/trecords.py 2007-10-27 18:25:05 UTC (rev 3468) +++ trunk/scipy/sandbox/timeseries/trecords.py 2007-10-27 20:44:00 UTC (rev 3469) @@ -255,9 +255,9 @@ for s in zip(*[getattr(self,f) for f in self.dtype.names])] return "[%s]" % ", ".join(mstr) else: - mstr = numeric.asarray(self._data.item(), dtype=object_) - mstr[list(self._fieldmask)] = masked_print_option - return str(mstr) + mstr = ["%s" % ",".join([str(i) for i in s]) + for s in zip([getattr(self,f) for f in self.dtype.names])] + return "(%s)" % ", ".join(mstr) def __repr__(self): """x.__repr__() <==> repr(x) @@ -522,3 +522,4 @@ print recfirst, type(recfirst) print mrec[0], type(mrec[0]) + Modified: trunk/scipy/sandbox/timeseries/tseries.py =================================================================== --- trunk/scipy/sandbox/timeseries/tseries.py 2007-10-27 18:25:05 UTC (rev 3468) +++ trunk/scipy/sandbox/timeseries/tseries.py 2007-10-27 20:44:00 UTC (rev 3469) @@ -25,7 +25,7 @@ from numpy.core.records import recarray from numpy.core.records import fromarrays as recfromarrays -import maskedarray as MA +import maskedarray from maskedarray import MaskedArray, MAError, masked, nomask, \ filled, getmask, getmaskarray, hsplit, make_mask_none, mask_or, make_mask, \ masked_array @@ -43,15 +43,21 @@ __all__ = [ 'TimeSeriesError','TimeSeriesCompatibilityError','TimeSeries','isTimeSeries', 'time_series', 'tsmasked', -'mask_period','mask_inside_period','mask_outside_period','compressed', -'adjust_endpoints','align_series','align_with','aligned','convert','group_byperiod', -'pct','tshift','fill_missing_dates', 'split', 'stack', 'concatenate_series', +'adjust_endpoints','align_series','align_with','aligned','asrecords', +'compressed','concatenate', 'concatenate_series','convert', +'day_of_week','day_of_year','day', 'empty_like', -'day_of_week','day_of_year','day','month','quarter','year', -'hour','minute','second', -'tofile','asrecords','flatten', -'first_unmasked_val', 'last_unmasked_val' - ] +'fill_missing_dates','first_unmasked_val','flatten', +'group_byperiod', +'hour', +'last_unmasked_val', +'mask_period','mask_inside_period','mask_outside_period','minute','month', +'pct', +'quarter', +'second','split', 'stack', +'tofile','tshift', +'year', +] def _unmasked_val(marray, x): "helper function for first_unmasked_val and last_unmasked_val" @@ -60,9 +66,9 @@ except AssertionError: raise ValueError("array must have ndim == 1") - idx = MA.extras.flatnotmasked_edges(marray) + idx = maskedarray.extras.flatnotmasked_edges(marray) if idx is None: - return MA.masked + return masked return marray[idx[x]] def first_unmasked_val(marray): @@ -1268,7 +1274,7 @@ tempData = masked_array(_values, mask=_mask) if tempData.ndim == 2 and func is not None: - tempData = MA.apply_along_axis(func, -1, tempData, *args, **kwargs) + tempData = maskedarray.apply_along_axis(func, -1, tempData, *args, **kwargs) newseries = tempData.view(type(series)) newseries._dates = date_array(start_date=start_date, length=len(newseries), @@ -1294,7 +1300,7 @@ obj = _convert1d(series, freq, func, position, *args, **kwargs) elif series.ndim == 2: base = _convert1d(series[:,0], freq, func, position, *args, **kwargs) - obj = MA.column_stack([_convert1d(m,freq,func,position, + obj = maskedarray.column_stack([_convert1d(m,freq,func,position, *args, **kwargs)._series for m in series.split()]).view(type(series)) obj._dates = base._dates @@ -1488,7 +1494,9 @@ for (new,old) in zip(newslc,oldslc): newdatad[new] = datad[old] newdatam[new] = datam[old] - newdata = MA.masked_array(newdatad, mask=newdatam, fill_value=fill_value) + if fill_value is None: + fill_value = getattr(data, 'fill_value', None) + newdata = maskedarray.masked_array(newdatad, mask=newdatam, fill_value=fill_value) _data = newdata.view(datat) _data._dates = newdates return _data @@ -1498,50 +1506,74 @@ resulting series has the same dates as each individual series. All series must be date compatible. -:Parameters: - `*series` : the series to be stacked +*Parameters*: + series : the series to be stacked """ _timeseriescompat_multiple(*series) - return time_series(MA.column_stack(series), series[0]._dates, + return time_series(maskedarray.column_stack(series), series[0]._dates, **_attrib_dict(series[0])) #............................................................................... def concatenate_series(*series, **kwargs): - """Concatenates a sequence of series, by chronological order. - Overlapping data are processed in a FIFO basis: the data from the first series - of the sequence will be overwritten by the data of the second series, and so forth. - If keep_gap is true, any gap between consecutive, non overlapping series are - kept: the corresponding data are masked. - """ + msg = """The use of this function is deprecated. +Please use concatenate instead. +Note: Please pay attention to the order of the series!""" + raise NameError(msg) + + +def concatenate(series, axis=0, remove_duplicates=True, fill_missing=False): + """Joins series together. - keep_gap = kwargs.pop('keep_gap', True) - if len(kwargs) > 0: - raise KeyError("unrecognized keyword: %s" % list(kwargs)[0]) +The series are joined in chronological order. Duplicated dates are handled with +the `remove_duplicates` parameter. If remove_duplicate=False, duplicated dates are +saved. Otherwise, only the first occurence of the date is conserved. + +Example +>>> a = time_series([1,2,3], start_date=today('D')) +>>> b = time_series([10,20,30], start_date=today('D')+1) +>>> c = concatenate((a,b)) +>>> c._series +masked_array(data = [ 1 2 3 30], + mask = False, + fill_value=999999) + - common_f = _compare_frequencies(*series) - start_date = min([s.start_date for s in series if s.start_date is not None]) - end_date = max([s.end_date for s in series if s.end_date is not None]) - newdtype = max([s.dtype for s in series]) - whichone = numeric.zeros((end_date-start_date+1), dtype=int_) - newseries = time_series(numeric.empty((end_date-start_date+1), dtype=newdtype), - dates=date_array(start_date, end_date, freq=common_f), - mask=True) - newdata = newseries._data - newmask = newseries._mask - for (k,s) in enumerate(series): - start = s.start_date - start_date - end = start + len(s) - whichone[start:end] = k+1 - newdata[start:end] = s._data - if s._mask is nomask: - newmask[start:end] = False - else: - newmask[start:end] = s._mask - keeper = whichone.astype(bool_) - if not keep_gap: - newseries = newseries[keeper] +*Parameters*: + series : {sequence} + Sequence of time series to join + axis : {integer} + Axis along which to join + remove_duplicates : boolean + Whether to remove duplicated dates. + fill_missing : {boolean} + Whether to fill the missing dates with missing values. + """ + # Get the common frequency, raise an error if incompatibility + common_f = _compare_frequencies(*series) + # Concatenate the order of series + sidx = numpy.concatenate([numpy.repeat(i,len(s)) + for (i,s) in enumerate(series)], axis=axis) + # Concatenate the dates and data + ndates = numpy.concatenate([s._dates for s in series], axis=axis) + ndata = maskedarray.concatenate([s._series for s in series], axis=axis) + # Resort the data chronologically + norder = ndates.argsort(kind='mergesort') + ndates = ndates[norder] + ndata = ndata[norder] + sidx = sidx[norder] + # + if not remove_duplicates: + ndates = date_array_fromlist(ndates, freq=common_f) + result = time_series(ndata, dates=ndates) else: - newdata[~keeper] = 0 - return newseries + # Find the original dates + orig = numpy.concatenate([[True],(numpy.diff(ndates) != 0)]) + result = time_series(ndata.compress(orig), + dates=ndates.compress(orig),freq=common_f) + if fill_missing: + result = fill_missing_dates(result) + return result + + #............................................................................... def empty_like(series): """Returns an empty series with the same dtype, mask and dates as series.""" @@ -1593,32 +1625,29 @@ "Make sure we're not losing the fill_value" dlist = ['2007-01-%02i' % i for i in range(1,16)] dates = date_array_fromlist(dlist) - series = time_series(MA.zeros(dates.shape), dates=dates, fill_value=-9999) + series = time_series(maskedarray.zeros(dates.shape), dates=dates, fill_value=-9999) assert_equal(series.fill_value, -9999) if 0: "Check time_series w/ an existing time series" dlist = ['2007-01-%02i' % i for i in range(1,16)] dates = date_array_fromlist(dlist) - series = time_series(MA.zeros(dates.shape), dates=dates, fill_value=-9999) + series = time_series(maskedarray.zeros(dates.shape), dates=dates, fill_value=-9999) newseries = time_series(series, fill_value=+9999) assert_equal(newseries._data, series._data) assert_equal(newseries._mask, series._mask) assert_equal(newseries.fill_value, +9999) - - if 0: - data = numpy.arange(5*24).reshape(5,24) + if 1: + "Check that the fill_value is kept" + data = [0,1,2,3,4,] datelist = ['2007-07-01','2007-07-02','2007-07-03','2007-07-05','2007-07-06'] dates = date_array_fromlist(datelist, 'D') -# dseries = time_series(data, dates) + dseries = time_series(data, dates, fill_value=-999) ndates = date_array_fromrange(start_date=dates[0],end_date=dates[-2]) - # - (A,B) = (data.ravel()[:4].reshape(2,2), dates[:-1]) - series = time_series(A,B) - fseries = fill_missing_dates(series) - assert_equal(fseries.shape, (5,)) - assert_equal(fseries._mask, [0,0,0,1,0,]) + fseries = fill_missing_dates(dseries) + assert_equal(dseries.fill_value, fseries.fill_value) + # - if 1: + if 0: dlist = ['2007-01-%02i' % i for i in (3,2,1)] data = [10,20,30] # series = time_series(data, dlist, mask=[1,0,0]) @@ -1626,7 +1655,7 @@ # series = time_series(data, dlist) series = time_series(data, dlist, mask=[1,0,0]) assert_equal(series._mask,[0,0,1]) - if 1: + if 0: dlist = ['2007-01-%02i' % i for i in range(1,16)] dates = date_array_fromlist(dlist) data = masked_array(numeric.arange(15), mask=[1,0,0,0,0]*3) @@ -1634,3 +1663,10 @@ empty_series = time_series([], freq='d') a, b = align_series(series, empty_series) + + if 1: + "Check concatenate..." + import dates + tt = time_series([.2,.2,.3],start_date=dates.Date('T',string='2007-10-10 01:10')) + tt._dates += [0, 9, 18] + From scipy-svn at scipy.org Sun Oct 28 02:15:29 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 28 Oct 2007 01:15:29 -0500 (CDT) Subject: [Scipy-svn] r3470 - trunk/scipy/sparse/tests Message-ID: <20071028061529.007BC39C0FE@new.scipy.org> Author: wnbell Date: 2007-10-28 01:15:25 -0500 (Sun, 28 Oct 2007) New Revision: 3470 Modified: trunk/scipy/sparse/tests/test_sparse.py Log: reduced runtime cost of a few unittests Modified: trunk/scipy/sparse/tests/test_sparse.py =================================================================== --- trunk/scipy/sparse/tests/test_sparse.py 2007-10-27 20:44:00 UTC (rev 3469) +++ trunk/scipy/sparse/tests/test_sparse.py 2007-10-28 06:15:25 UTC (rev 3470) @@ -559,59 +559,58 @@ #check conversions for x in self.dtypes: for y in self.dtypes: - for z in self.dtypes: - A = self.A.astype(x) - B = self.B.astype(y) - C = self.C.astype(z) - - Asp = self.spmatrix(A) - Bsp = self.spmatrix(B) - Csp = self.spmatrix(C) + A = self.A.astype(x) + B = self.B.astype(y) + C = self.C.astype(y) + + Asp = self.spmatrix(A) + Bsp = self.spmatrix(B) + Csp = self.spmatrix(C) - #addition - D1 = A + B - D2 = A + C - D3 = B + C - S1 = Asp + Bsp - S2 = Asp + Csp - S3 = Bsp + Csp - - assert_equal(D1.dtype,S1.dtype) - assert_equal(D2.dtype,S2.dtype) - assert_equal(D3.dtype,S3.dtype) - assert_array_equal(D1,S1.todense()) - assert_array_equal(D2,S2.todense()) - assert_array_equal(D3,S3.todense()) - assert_array_equal(D1,Asp + B) #check sparse + dense - assert_array_equal(D2,Asp + C) - assert_array_equal(D3,Bsp + C) - assert_array_equal(D1,A + Bsp) #check dense + sparse - assert_array_equal(D2,A + Csp) - assert_array_equal(D3,B + Csp) + #addition + D1 = A + B + D2 = A + C + D3 = B + C + S1 = Asp + Bsp + S2 = Asp + Csp + S3 = Bsp + Csp + + assert_equal(D1.dtype,S1.dtype) + assert_equal(D2.dtype,S2.dtype) + assert_equal(D3.dtype,S3.dtype) + assert_array_equal(D1,S1.todense()) + assert_array_equal(D2,S2.todense()) + assert_array_equal(D3,S3.todense()) + assert_array_equal(D1,Asp + B) #check sparse + dense + assert_array_equal(D2,Asp + C) + assert_array_equal(D3,Bsp + C) + assert_array_equal(D1,A + Bsp) #check dense + sparse + assert_array_equal(D2,A + Csp) + assert_array_equal(D3,B + Csp) - #subtraction - D1 = A - B - D2 = A - C - D3 = B - C - S1 = Asp - Bsp - S2 = Asp - Csp - S3 = Bsp - Csp - - assert_equal(D1.dtype,S1.dtype) - assert_equal(D2.dtype,S2.dtype) - assert_equal(D3.dtype,S3.dtype) - assert_array_equal(D1,S1.todense()) - assert_array_equal(D2,S2.todense()) - assert_array_equal(D3,S3.todense()) - assert_array_equal(D1,Asp - B) #check sparse - dense - assert_array_equal(D2,Asp - C) - assert_array_equal(D3,Bsp - C) - assert_array_equal(D1,A - Bsp) #check dense - sparse - try: - assert_array_equal(D2,A - Csp) - except: - import pdb; pdb.set_trace() - assert_array_equal(D3,B - Csp) + #subtraction + D1 = A - B + D2 = A - C + D3 = B - C + S1 = Asp - Bsp + S2 = Asp - Csp + S3 = Bsp - Csp + + assert_equal(D1.dtype,S1.dtype) + assert_equal(D2.dtype,S2.dtype) + assert_equal(D3.dtype,S3.dtype) + assert_array_equal(D1,S1.todense()) + assert_array_equal(D2,S2.todense()) + assert_array_equal(D3,S3.todense()) + assert_array_equal(D1,Asp - B) #check sparse - dense + assert_array_equal(D2,Asp - C) + assert_array_equal(D3,Bsp - C) + assert_array_equal(D1,A - Bsp) #check dense - sparse + try: + assert_array_equal(D2,A - Csp) + except: + import pdb; pdb.set_trace() + assert_array_equal(D3,B - Csp) def check_mu(self): @@ -621,32 +620,30 @@ assert_array_equal(self.A*self.B.T,(self.Asp*self.Bsp.T).todense()) assert_array_equal(self.A*self.C.T,(self.Asp*self.Csp.T).todense()) - #check conversions for x in self.dtypes: for y in self.dtypes: - for z in self.dtypes: - A = self.A.astype(x) - B = self.B.astype(y) - C = self.C.astype(z) - - Asp = self.spmatrix(A) - Bsp = self.spmatrix(B) - Csp = self.spmatrix(C) + A = self.A.astype(x) + B = self.B.astype(y) + C = self.C.astype(y) + + Asp = self.spmatrix(A) + Bsp = self.spmatrix(B) + Csp = self.spmatrix(C) - D1 = A * B.T - D2 = A * C.T - D3 = B * C.T + D1 = A * B.T + D2 = A * C.T + D3 = B * C.T - S1 = Asp * Bsp.T - S2 = Asp * Csp.T - S3 = Bsp * Csp.T + S1 = Asp * Bsp.T + S2 = Asp * Csp.T + S3 = Bsp * Csp.T - assert_array_equal(D1,S1.todense()) - assert_array_equal(D2,S2.todense()) - assert_array_equal(D3,S3.todense()) - assert_equal(D1.dtype,S1.dtype) - assert_equal(D2.dtype,S2.dtype) - assert_equal(D3.dtype,S3.dtype) + assert_array_equal(D1,S1.todense()) + assert_array_equal(D2,S2.todense()) + assert_array_equal(D3,S3.todense()) + assert_equal(D1.dtype,S1.dtype) + assert_equal(D2.dtype,S2.dtype) + assert_equal(D3.dtype,S3.dtype) From scipy-svn at scipy.org Sun Oct 28 04:07:42 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 28 Oct 2007 03:07:42 -0500 (CDT) Subject: [Scipy-svn] r3471 - trunk/scipy/sparse Message-ID: <20071028080742.BBD2D39C01A@new.scipy.org> Author: wnbell Date: 2007-10-28 03:07:38 -0500 (Sun, 28 Oct 2007) New Revision: 3471 Modified: trunk/scipy/sparse/sparse.py Log: unified format check for CSR and CSC matrices functionality is also exposed as check_format() Modified: trunk/scipy/sparse/sparse.py =================================================================== --- trunk/scipy/sparse/sparse.py 2007-10-28 06:15:25 UTC (rev 3470) +++ trunk/scipy/sparse/sparse.py 2007-10-28 08:07:38 UTC (rev 3471) @@ -299,8 +299,6 @@ return self._imag() elif attr == 'size': return self.getnnz() -# elif attr == 'ftype': -# return _transtabl.get(self.dtype.char,'') else: raise AttributeError, attr + " not found" @@ -443,19 +441,13 @@ m, n = self.shape if axis == 0: # sum over columns - # Does the following multiplication work in NumPy now? - o = asmatrix(ones((1, m), dtype=self.dtype)) - return o * self - # o = ones(m, dtype=self.dtype) - # return asmatrix(self.rmatvec(o)) + return asmatrix(ones((1, m), dtype=self.dtype)) * self elif axis == 1: # sum over rows - o = asmatrix(ones((n, 1), dtype=self.dtype)) - return self * o + return self * asmatrix(ones((n, 1), dtype=self.dtype)) elif axis == None: # sum over rows and columns - o1 = asmatrix(ones((n, 1), dtype=self.dtype)) - return (self * o1).sum() + return ( self * asmatrix(ones((n, 1), dtype=self.dtype)) ).sum() else: raise ValueError, "axis out of bounds" @@ -513,7 +505,23 @@ fd.close() class _cs_matrix(spmatrix): - def _check(self): + """base matrix class for compressed row and column oriented matrices""" + def _check_format(self, orientation, full_check): + # some functions pass floats + self.shape = tuple([int(x) for x in self.shape]) + + assert(orientation in ['row','column']) + if orientation == 'row': + primary,secondary = 'row','column' + indptr_size = self.shape[0] + 1 + indices_bound = self.shape[1] + else: + primary,secondary = 'column','row' + indptr_size = self.shape[1] + 1 + indices_bound = self.shape[0] + + + # index arrays should have integer data types if self.indptr.dtype.kind != 'i': warnings.warn("indptr array has non-integer dtype. " \ "Casting from %s to int" % self.indptr.dtype.name ) @@ -527,8 +535,49 @@ self.indptr = to_native(self.indptr) self.indices = to_native(self.indices) self.data = to_native(self.data) + + # set the data type + self.dtype = self.data.dtype + + + # check array shapes + if (rank(self.data) != 1) or (rank(self.indices) != 1) or \ + (rank(self.indptr) != 1): + raise ValueError,"data, indices, and indptr should be rank 1" - #TODO unify csr and csr _check + # check index pointer + if (len(self.indptr) != indptr_size ): + raise ValueError, \ + "index pointer size (%d) should be (%d)" % \ + (len(self.indptr), indptr_size) + if (self.indptr[0] != 0): + raise ValueError,"index pointer should start with 0" + + # check index and data arrays + if (len(self.indices) != len(self.data)): + raise ValueError,"indices and data should have the same size" + if (self.indptr[-1] > len(self.indices)): + raise ValueError, \ + "Last value of index pointer should be less than "\ + "the size of index and data arrays" + + self.nnz = self.indptr[-1] + self.nzmax = len(self.indices) + + if full_check: + #check format validity (more expensive) + if self.nnz > 0: + if amax(self.indices[:self.nnz]) >= indices_bound: + raise ValueError, "%s index values must be < %d" % \ + (secondary,indices_bound) + if amin(self.indices[:self.nnz]) < 0: + raise ValueError, "%s index values must be >= 0" % \ + secondary + if numpy.diff(self.indptr).min() < 0: + raise ValueError,'index pointer values must form a " \ + "non-decreasing sequence' + + def astype(self, t): return self._with_data(self.data.astype(t)) @@ -541,17 +590,16 @@ _formats[format][1])) def _with_data(self,data,copy=True): - """ - Return a matrix with the same sparsity structure as self, + """Returns a matrix with the same sparsity structure as self, but with different data. By default the structure arrays (i.e. .indptr and .indices) are copied. """ if copy: return self.__class__((data,self.indices.copy(),self.indptr.copy()), \ - dims=self.shape,dtype=data.dtype,check=False) + dims=self.shape,dtype=data.dtype) else: return self.__class__((data,self.indices,self.indptr), \ - dims=self.shape,dtype=data.dtype,check=False) + dims=self.shape,dtype=data.dtype) def __abs__(self): return self._with_data(abs(self.data)) @@ -574,7 +622,7 @@ indptr, ind, data = fn(in_shape[0], in_shape[1], \ self.indptr, self.indices, self.data, other.indptr, other.indices, other.data) - return self.__class__((data, ind, indptr), dims=out_shape, check=False) + return self.__class__((data, ind, indptr), dims=out_shape) def __add__(self,other,fn): @@ -780,7 +828,7 @@ def _transpose(self, cls, copy=False): M, N = self.shape - return cls((self.data,self.indices,self.indptr),(N,M),copy=copy,check=False) + return cls((self.data,self.indices,self.indptr),(N,M),copy=copy) def conj(self, copy=False): @@ -862,8 +910,9 @@ - csc_matrix((data, row, ptr), [(M, N)]) standard CSC representation """ - def __init__(self, arg1, dims=None, nzmax=NZMAX, dtype=None, copy=False, check=True): + def __init__(self, arg1, dims=None, nzmax=NZMAX, dtype=None, copy=False): _cs_matrix.__init__(self) + if isdense(arg1): self.dtype = getdtype(dtype, arg1) # Convert the dense array or matrix arg1 to CSC format @@ -973,64 +1022,20 @@ self.shape = (M, N) - self._check(check) + self.check_format(full_check=False) + def check_format(self,full_check=True): + """check whether matrix is in valid CSC format - def _check(self,full_check=True): - _cs_matrix._check(self) - - # some functions pass floats - self.shape = tuple([int(x) for x in self.shape]) + *Parameters*: + full_check: + True - rigorous check, O(N) operations : default + False - basic check, O(1) operations - M, N = self.shape - nnz = self.indptr[-1] - nzmax = len(self.indices) - if (rank(self.data) != 1) or (rank(self.indices) != 1) or \ - (rank(self.indptr) != 1): - raise ValueError, "data, rowind, and indptr arrays "\ - "should be rank 1" - if (len(self.data) != nzmax): - raise ValueError, "data and row list should have same length" - if (self.indptr[0] != 0): - raise ValueError,"index pointer should start with 0" - if (len(self.indptr) != N+1): - raise ValueError, \ - "index pointer size (%d) should be N+1 (%d)" %\ - (len(self.indptr), N+1) - if (nzmax < nnz): - raise ValueError, "nzmax (%d) must not be less than nnz (%d)" %\ - (nzmax, nnz) + """ - if full_check: - #check format validity (more expensive) - if nnz > 0: - if amax(self.indices[:nnz]) >= M: - raise ValueError, "row values must be < M" - if amin(self.indices[:nnz]) < 0: - raise ValueError, "row values must be >= 0" - if numpy.diff(self.indptr).min() < 0: - raise ValueError,'indptr values must form a non-decreasing sequence' + _cs_matrix._check_format(self,'column',full_check) - if (self.indptr[-1] > len(self.indices)): - raise ValueError, \ - "Last value of index list should be less than "\ - "the size of data list" - - - #TODO remove - #if (self.indices.dtype != numpy.intc): - # self.indices = self.indices.astype(numpy.intc) - #if (self.indptr.dtype != numpy.intc): - # self.indptr = self.indptr.astype(numpy.intc) - - self.nnz = nnz - self.nzmax = nzmax - self.dtype = self.data.dtype - #TODO remove - #if self.dtype.char not in 'fdFD': - # self.data = 1.0 * self.data - # self.dtype = self.data.dtype - def __getattr__(self, attr): if attr == 'rowind': warnings.warn("rowind attribute no longer in use. Use .indices instead", @@ -1144,7 +1149,7 @@ else: raise IndexError, "row index occurs more than once" - self._check() + self.check_format(full_check=False) else: # We should allow slices here! raise IndexError, "invalid index" @@ -1172,7 +1177,7 @@ def tocsr(self): indptr, colind, data = csctocsr(self.shape[0], self.shape[1], \ self.indptr, self.indices, self.data) - return csr_matrix((data, colind, indptr), self.shape, check=False) + return csr_matrix((data, colind, indptr), self.shape) def _toother(self): return self.tocsr() @@ -1195,7 +1200,6 @@ self.data = self.data[:nnz] self.indices = self.indices[:nnz] self.nzmax = nnz - self._check() def ensure_sorted_indices(self, inplace=False): """Return a copy of this matrix where the row indices are sorted @@ -1232,7 +1236,7 @@ - csr_matrix((data, col, ptr), [dims=(M, N)]) standard CSR representation """ - def __init__(self, arg1, dims=None, nzmax=NZMAX, dtype=None, copy=False, check=True): + def __init__(self, arg1, dims=None, nzmax=NZMAX, dtype=None, copy=False): _cs_matrix.__init__(self) if isdense(arg1): self.dtype = getdtype(dtype, arg1) @@ -1336,58 +1340,21 @@ N = max(oldN, N) self.shape = (M, N) - - self._check(check) - def _check(self,full_check=True): - _cs_matrix._check(self) + self.check_format(full_check=False) - # some functions pass floats - self.shape = tuple([int(x) for x in self.shape]) + def check_format(self,full_check=True): + """check whether matrix is in valid CSR format - M, N = self.shape - nnz = self.indptr[-1] - nzmax = len(self.indices) - if (rank(self.data) != 1) or (rank(self.indices) != 1) or \ - (rank(self.indptr) != 1): - raise ValueError, "data, colind, and indptr arrays "\ - "should be rank 1" - if (len(self.data) != nzmax): - raise ValueError, "data and row list should have same length" - if (self.indptr[0] != 0): - raise ValueError,"index pointer should start with 0" - if (len(self.indptr) != M+1): - raise ValueError, "index pointer should be of length #rows + 1" + *Parameters*: + full_check: + True - perform rigorous checking - default + False - perform basic format check + """ - if full_check: - #check format validity (more expensive) - if nnz > 0: - if amax(self.indices[:nnz]) >= N: - raise ValueError, "column values must be < N" - if amin(self.indices[:nnz]) < 0: - raise ValueError, "column values must be >= 0" - if numpy.diff(self.indptr).min() < 0: - raise ValueError,'indptr values must form a non-decreasing sequence' + _cs_matrix._check_format(self,'row',full_check) - if (nnz > nzmax): - raise ValueError, \ - "last value of index list should be less than "\ - "the size of data list" - #TODO remove this - #if (self.indices.dtype != numpy.intc): - # self.indices = self.indices.astype(numpy.intc) - #if (self.indptr.dtype != numpy.intc): - # self.indptr = self.indptr.astype(numpy.intc) - - self.nnz = nnz - self.nzmax = nzmax - self.dtype = self.data.dtype - #TODO remove - #if self.dtype.char not in 'fdFD': - # self.data = self.data + 0.0 - # self.dtype = self.data.dtype - def __getattr__(self, attr): if attr == 'colind': warnings.warn("colind attribute no longer in use. Use .indices instead", @@ -1501,7 +1468,7 @@ else: raise IndexError, "row index occurs more than once" - self._check() + self.check_format(full_check=False) else: # We should allow slices here! raise IndexError, "invalid index" @@ -1520,7 +1487,7 @@ def tocsc(self): indptr, rowind, data = csrtocsc(self.shape[0], self.shape[1], \ self.indptr, self.indices, self.data) - return csc_matrix((data, rowind, indptr), self.shape, check=False) + return csc_matrix((data, rowind, indptr), self.shape) def _toother(self): return self.tocsc() @@ -1546,7 +1513,6 @@ self.data = self.data[:nnz] self.indices = self.indices[:nnz] self.nzmax = nnz - self._check() def ensure_sorted_indices(self, inplace=False): """Return a copy of this matrix where the column indices are sorted @@ -2276,22 +2242,22 @@ if (nnz != len(self.row)) or (nnz != len(self.col)): raise ValueError, "row, column, and data array must all be "\ "the same length" + + # index arrays should have integer data types + if self.row.dtype.kind != 'i': + warnings.warn("row index array has non-integer dtype. " \ + "Casting from %s to int" % self.row.dtype.name ) + self.row = self.row.astype('i') + if self.col.dtype.kind != 'i': + warnings.warn("column index array has non-integer dtype. " \ + "Casting from %s to int" % self.col.dtype.name ) + self.col = self.col.astype('i') - if self.row.dtype > numpy.dtype('int64'): - raise TypeError,'row array has invalid dtype' - if self.col.dtype > numpy.dtype('int64'): - raise TypeError,'column array has invalid dtype' - - #TODO fix this bandaid + #TODO do this in SWIG self.row = to_native(self.row) self.col = to_native(self.col) self.data = to_native(self.data) - #if (self.row.dtype != numpy.intc): - # self.row = self.row.astype(numpy.intc) - #if (self.col.dtype != numpy.intc): - # self.col = self.col.astype(numpy.intc) - if nnz > 0: if(amax(self.row) >= self.shape[0]): raise ValueError, "row index exceedes matrix dimensions" @@ -2340,7 +2306,7 @@ indptr, rowind, data = cootocsc(self.shape[0], self.shape[1], \ self.nnz, self.row, self.col, \ self.data) - return csc_matrix((data, rowind, indptr), self.shape, check=True) + return csc_matrix((data, rowind, indptr), self.shape) def tocsr(self): @@ -2350,7 +2316,7 @@ indptr, colind, data = cootocsr(self.shape[0], self.shape[1], \ self.nnz, self.row, self.col, \ self.data) - return csr_matrix((data, colind, indptr), self.shape, check=False) + return csr_matrix((data, colind, indptr), self.shape) def tocoo(self, copy=False): return self.toself(copy) From scipy-svn at scipy.org Sun Oct 28 23:04:29 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Sun, 28 Oct 2007 22:04:29 -0500 (CDT) Subject: [Scipy-svn] r3472 - in trunk/scipy/sandbox/timeseries: . tests Message-ID: <20071029030429.81BE139C049@new.scipy.org> Author: mattknox_ca Date: 2007-10-28 22:03:54 -0500 (Sun, 28 Oct 2007) New Revision: 3472 Modified: trunk/scipy/sandbox/timeseries/tests/test_timeseries.py trunk/scipy/sandbox/timeseries/tseries.py Log: removed some obsolete parameters from time_series function and cleaned up the corresponding __doc__ string Modified: trunk/scipy/sandbox/timeseries/tests/test_timeseries.py =================================================================== --- trunk/scipy/sandbox/timeseries/tests/test_timeseries.py 2007-10-28 08:07:38 UTC (rev 3471) +++ trunk/scipy/sandbox/timeseries/tests/test_timeseries.py 2007-10-29 03:03:54 UTC (rev 3472) @@ -53,7 +53,7 @@ def test_fromrange (self): "Base data definition." (dlist, dates, data) = self.d - series = time_series(data, start_date=dates[0], length=15) + series = time_series(data, start_date=dates[0]) assert(isinstance(series, TimeSeries)) assert_equal(series._mask, [1,0,0,0,0]*3) assert_equal(series._series, data) @@ -530,28 +530,32 @@ def test__timeseriescompat_multiple(self): "Tests the compatibility of multiple time series." - seriesM_10 = time_series(numpy.arange(10), - date_array( - start_date=Date(freq='m', year=2005, month=1), - length=10) + seriesM_10 = time_series( + numpy.arange(10), + date_array( + start_date=Date(freq='m', year=2005, month=1), + length=10) ) - seriesD_10 = time_series(numpy.arange(10), - date_array( - start_date=Date(freq='d', year=2005, month=1, day=1), - length=10) + seriesD_10 = time_series( + numpy.arange(10), + date_array( + start_date=Date(freq='d', year=2005, month=1, day=1), + length=10) ) - seriesD_5 = time_series(numpy.arange(5), - date_array( - start_date=Date(freq='d', year=2005, month=1, day=1), - length=5) + seriesD_5 = time_series( + numpy.arange(5), + date_array( + start_date=Date(freq='d', year=2005, month=1, day=1), + length=5) ) - seriesD_5_apr = time_series(numpy.arange(5), - date_array( - start_date=Date(freq='d', year=2005, month=4, day=1), - length=5) + seriesD_5_apr = time_series( + numpy.arange(5), + date_array( + start_date=Date(freq='d', year=2005, month=4, day=1), + length=5) ) assert(tseries._timeseriescompat_multiple(seriesM_10, seriesM_10, seriesM_10)) Modified: trunk/scipy/sandbox/timeseries/tseries.py =================================================================== --- trunk/scipy/sandbox/timeseries/tseries.py 2007-10-28 08:07:38 UTC (rev 3471) +++ trunk/scipy/sandbox/timeseries/tseries.py 2007-10-29 03:03:54 UTC (rev 3472) @@ -360,12 +360,11 @@ construction as it allows greater flexibility and convenience. """ def __new__(cls, data, dates, mask=nomask, dtype=None, copy=False, - fill_value=None, subok=True, keep_mask=True, small_mask=True, - hard_mask=False, **options): + fill_value=None, subok=True, keep_mask=True, hard_mask=False, + **options): maparms = dict(copy=copy, dtype=dtype, fill_value=fill_value, subok=subok, - keep_mask=keep_mask, small_mask=small_mask, - hard_mask=hard_mask) + keep_mask=keep_mask, hard_mask=hard_mask) _data = MaskedArray(data, mask=mask, **maparms) # Get the dates ...................................................... @@ -945,21 +944,37 @@ ##### ------------------------------------------------------------------------- #---- --- TimeSeries creator --- ##### ------------------------------------------------------------------------- -def time_series(data, dates=None, freq=None, start_date=None, end_date=None, - length=None, mask=nomask, dtype=None, copy=False, - fill_value=None, keep_mask=True, small_mask=True, +def time_series(data, dates=None, start_date=None, freq=None, mask=nomask, + dtype=None, copy=False, fill_value=None, keep_mask=True, hard_mask=False): """Creates a TimeSeries object -:Parameters: - `dates` : ndarray - Array of dates. - `data` : - Array of data. - """ +*Parameters*: + data : {array_like} + data portion of the array. Any data that is valid for constructing a + MaskedArray can be used here. May also be a TimeSeries object + dates : {DateArray}, optional + Date part. + freq : {freq_spec}, optional + a valid frequency specification + start_date : {Date}, optional + date corresponding to index 0 in the data + +*Other Parameters*: + All other parameters that are accepted by the *array* function in the + maskedarray module are also accepted by this function. + +*Notes*: + the date portion of the time series must be specified in one of the + following ways: + - specify a TimeSeries object for the *data* parameter + - pass a DateArray for the *dates* parameter + - specify a start_date (a continuous DateArray will be automatically + constructed for the dates portion) + - specify just a frequency (for TimeSeries of size zero) +""" maparms = dict(copy=copy, dtype=dtype, fill_value=fill_value, subok=True, - keep_mask=keep_mask, small_mask=small_mask, - hard_mask=hard_mask,) + keep_mask=keep_mask, hard_mask=hard_mask,) data = masked_array(data, mask=mask, **maparms) freq = check_freq(freq) @@ -980,11 +995,8 @@ else: dshape = data.shape if len(dshape) > 0: - if length is None: - length = dshape[0] - if len(dshape) > 0: - _dates = date_array(start_date=start_date, end_date=end_date, - length=length, freq=freq) + length = dshape[0] + _dates = date_array(start_date=start_date, freq=freq, length=length) else: _dates = date_array([], freq=freq) @@ -995,7 +1007,7 @@ return TimeSeries(data=data, dates=_dates, mask=data._mask, copy=copy, dtype=dtype, fill_value=fill_value, keep_mask=keep_mask, - small_mask=small_mask, hard_mask=hard_mask,) + hard_mask=hard_mask,) def isTimeSeries(series): From scipy-svn at scipy.org Tue Oct 30 18:44:03 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Tue, 30 Oct 2007 17:44:03 -0500 (CDT) Subject: [Scipy-svn] r3474 - in trunk/scipy/sandbox/timeseries: include lib src Message-ID: <20071030224403.6D01C39C04F@new.scipy.org> Author: mattknox_ca Date: 2007-10-30 17:43:58 -0500 (Tue, 30 Oct 2007) New Revision: 3474 Modified: trunk/scipy/sandbox/timeseries/include/c_tseries.h trunk/scipy/sandbox/timeseries/lib/moving_funcs.py trunk/scipy/sandbox/timeseries/src/c_tseries.c trunk/scipy/sandbox/timeseries/src/cseries.c Log: added mov_max and mov_min functions to moving_funcs Modified: trunk/scipy/sandbox/timeseries/include/c_tseries.h =================================================================== --- trunk/scipy/sandbox/timeseries/include/c_tseries.h 2007-10-29 15:18:13 UTC (rev 3473) +++ trunk/scipy/sandbox/timeseries/include/c_tseries.h 2007-10-30 22:43:58 UTC (rev 3474) @@ -7,6 +7,8 @@ PyObject *MaskedArray_mov_sum(PyObject *, PyObject *, PyObject *); PyObject *MaskedArray_mov_median(PyObject *, PyObject *, PyObject *); +PyObject *MaskedArray_mov_min(PyObject *, PyObject *, PyObject *); +PyObject *MaskedArray_mov_max(PyObject *, PyObject *, PyObject *); PyObject *MaskedArray_mov_average(PyObject *, PyObject *, PyObject *); PyObject *MaskedArray_mov_stddev(PyObject *, PyObject *, PyObject *); Modified: trunk/scipy/sandbox/timeseries/lib/moving_funcs.py =================================================================== --- trunk/scipy/sandbox/timeseries/lib/moving_funcs.py 2007-10-29 15:18:13 UTC (rev 3473) +++ trunk/scipy/sandbox/timeseries/lib/moving_funcs.py 2007-10-30 22:43:58 UTC (rev 3474) @@ -11,6 +11,12 @@ __revision__ = "$Revision: 2819 $" __date__ = '$Date: 2007-03-03 18:00:20 -0500 (Sat, 03 Mar 2007) $' +__all__ = ['mov_sum', 'mov_median', 'mov_min', 'mov_max', + 'mov_average', 'mov_mean', 'mov_average_expw', + 'mov_stddev', 'mov_var', 'mov_covar', 'mov_corr', + 'cmov_average', 'cmov_mean', 'cmov_window' + ] + import numpy as N from numpy import bool_, float_ narray = N.array @@ -22,14 +28,8 @@ marray = MA.array from timeseries.cseries import MA_mov_stddev, MA_mov_sum, MA_mov_average, \ - MA_mov_median + MA_mov_median, MA_mov_min, MA_mov_max -__all__ = ['mov_sum', 'mov_median', - 'mov_average', 'mov_mean', 'mov_average_expw', - 'mov_stddev', 'mov_var', 'mov_covar', 'mov_corr', - 'cmov_average', 'cmov_mean', 'cmov_window' - ] - def _process_result_dict(orig_data, result_dict): "process the results from the c function" @@ -101,6 +101,34 @@ return _moving_func(data, MA_mov_median, kwargs) #............................................................................... +def mov_min(data, span, dtype=None): + """Calculates the moving minimum of a series. + +*Parameters*: + $$data$$ + $$span$$ + $$dtype$$""" + + kwargs = {'span':span} + if dtype is not None: + kwargs['dtype'] = dtype + + return _moving_func(data, MA_mov_min, kwargs) +#............................................................................... +def mov_max(data, span, dtype=None): + """Calculates the moving max of a series. + +*Parameters*: + $$data$$ + $$span$$ + $$dtype$$""" + + kwargs = {'span':span} + if dtype is not None: + kwargs['dtype'] = dtype + + return _moving_func(data, MA_mov_max, kwargs) +#............................................................................... def mov_average(data, span, dtype=None): """Calculates the moving average of a series. Modified: trunk/scipy/sandbox/timeseries/src/c_tseries.c =================================================================== --- trunk/scipy/sandbox/timeseries/src/c_tseries.c 2007-10-29 15:18:13 UTC (rev 3473) +++ trunk/scipy/sandbox/timeseries/src/c_tseries.c 2007-10-30 22:43:58 UTC (rev 3474) @@ -552,17 +552,10 @@ } -/* computation portion of moving median. Appropriate mask is overlayed on top - afterwards. - The algorithm used here is based on the code found at: - http://cran.r-project.org/src/contrib/Devel/runStat_1.1.tar.gz - - This code was originally released under the GPL, but the author - (David Brahm) has granted me (and scipy) permission to use it under the BSD - license. */ +//calc_mov_median(PyArrayObject *orig_ndarray, int span, int rtype) PyObject* -calc_mov_median(PyArrayObject *orig_ndarray, int span, int rtype) +calc_mov_ranked(PyArrayObject *orig_ndarray, int span, int rtype, char rank_type) { PyArrayObject *result_ndarray=NULL; PyObject **result_array, **ref_array, **even_array=NULL; @@ -604,13 +597,29 @@ r[i] = 1; } - if ((span % 2) == 0) { + if (rank_type == 'E' && ((span % 2) == 0)) { // array to store two median values when span is an even # even_array = calloc(2, sizeof(PyObject*)); MEM_CHECK(even_array) } - R = (span + 1)/2; + switch(rank_type) { + case 'E': // median + R = (span + 1)/2; + break; + case 'I': // min + R = 1; + break; + case 'A': // max + R = span; + break; + default: + { + PyErr_SetString(PyExc_RuntimeError, "unexpected rank type"); + return NULL; + } + } + one_half = PyFloat_FromDouble(0.5); z = arr_size - span; @@ -751,8 +760,8 @@ rtype = _get_type_num(((PyArrayObject*)orig_ndarray)->descr, dtype); } - result_ndarray = calc_mov_median((PyArrayObject*)orig_ndarray, - span, rtype); + result_ndarray = calc_mov_ranked((PyArrayObject*)orig_ndarray, + span, rtype, 'E'); ERR_CHECK(result_ndarray) result_dict = PyDict_New(); @@ -765,8 +774,77 @@ return result_dict; } +PyObject * +MaskedArray_mov_min(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *orig_arrayobj=NULL, *orig_ndarray=NULL, + *result_ndarray=NULL, *result_mask=NULL, *result_dict=NULL; + PyArray_Descr *dtype=NULL; + int rtype, span; + + static char *kwlist[] = {"array", "span", "dtype", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "Oi|O&:mov_min(array, span, dtype)", kwlist, + &orig_arrayobj, &span, + PyArray_DescrConverter2, &dtype)) return NULL; + + check_mov_args(orig_arrayobj, span, 1, + &orig_ndarray, &result_mask); + + rtype = _get_type_num(((PyArrayObject*)orig_ndarray)->descr, dtype); + + result_ndarray = calc_mov_ranked((PyArrayObject*)orig_ndarray, + span, rtype, 'I'); + ERR_CHECK(result_ndarray) + + result_dict = PyDict_New(); + MEM_CHECK(result_dict) + PyDict_SetItemString(result_dict, "array", result_ndarray); + PyDict_SetItemString(result_dict, "mask", result_mask); + + Py_DECREF(result_ndarray); + Py_DECREF(result_mask); + return result_dict; +} + PyObject * +MaskedArray_mov_max(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *orig_arrayobj=NULL, *orig_ndarray=NULL, + *result_ndarray=NULL, *result_mask=NULL, *result_dict=NULL; + PyArray_Descr *dtype=NULL; + + int rtype, span; + + static char *kwlist[] = {"array", "span", "dtype", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "Oi|O&:mov_max(array, span, dtype)", kwlist, + &orig_arrayobj, &span, + PyArray_DescrConverter2, &dtype)) return NULL; + + check_mov_args(orig_arrayobj, span, 1, + &orig_ndarray, &result_mask); + + rtype = _get_type_num(((PyArrayObject*)orig_ndarray)->descr, dtype); + + result_ndarray = calc_mov_ranked((PyArrayObject*)orig_ndarray, + span, rtype, 'A'); + ERR_CHECK(result_ndarray) + + result_dict = PyDict_New(); + MEM_CHECK(result_dict) + PyDict_SetItemString(result_dict, "array", result_ndarray); + PyDict_SetItemString(result_dict, "mask", result_mask); + + Py_DECREF(result_ndarray); + Py_DECREF(result_mask); + return result_dict; +} + +PyObject * MaskedArray_mov_stddev(PyObject *self, PyObject *args, PyObject *kwds) { Modified: trunk/scipy/sandbox/timeseries/src/cseries.c =================================================================== --- trunk/scipy/sandbox/timeseries/src/cseries.c 2007-10-29 15:18:13 UTC (rev 3473) +++ trunk/scipy/sandbox/timeseries/src/cseries.c 2007-10-30 22:43:58 UTC (rev 3474) @@ -8,6 +8,10 @@ METH_VARARGS | METH_KEYWORDS, ""}, {"MA_mov_median", (PyCFunction)MaskedArray_mov_median, METH_VARARGS | METH_KEYWORDS, ""}, + {"MA_mov_min", (PyCFunction)MaskedArray_mov_min, + METH_VARARGS | METH_KEYWORDS, ""}, + {"MA_mov_max", (PyCFunction)MaskedArray_mov_max, + METH_VARARGS | METH_KEYWORDS, ""}, {"MA_mov_average", (PyCFunction)MaskedArray_mov_average, METH_VARARGS | METH_KEYWORDS, ""}, {"MA_mov_stddev", (PyCFunction)MaskedArray_mov_stddev, From scipy-svn at scipy.org Wed Oct 31 14:02:41 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 31 Oct 2007 13:02:41 -0500 (CDT) Subject: [Scipy-svn] r3475 - in trunk/scipy/io: . tests Message-ID: <20071031180241.B766439C231@new.scipy.org> Author: chris.burns Date: 2007-10-31 13:02:38 -0500 (Wed, 31 Oct 2007) New Revision: 3475 Added: trunk/scipy/io/tests/test_datasource.py Modified: trunk/scipy/io/datasource.py Log: Rewrite datasource, add testing module. Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-30 22:43:58 UTC (rev 3474) +++ trunk/scipy/io/datasource.py 2007-10-31 18:02:38 UTC (rev 3475) @@ -1,20 +1,34 @@ -"""A generic interface for importing data. Data sources can originate from -URLs or local paths and can be in compressed or uncompressed form. +"""A generic file interface for handling data files. The goal of datasource is +to abstract some of the file system operations when dealing with files. +Specifically acquiring the files. DataSource files can originate locally or +remotely, for example: + local files - '/home/name/blah/blah/blah/data.txt' + URLs (http, ftp, ...) - 'http://www.scipy.org/not/real/data.txt' -""" +DataSource files can also be compressed or uncompressed. Currently only gzip +and bz2 are supported. -# TODO: Make DataSource and Repository the public interface. -# _Cache will be used internally. Add methods in DataSource to expose -# some of the functionality that exists only in _Cache currently. +In a typical use, you would pass a DataSource to a function that would open +up the DataSource (as it would any file-like object) and read the file. +Ex: + >>> ds = datasource.DataSource() + >>> fp = ds.open('http://www.scipy.org/not/real/data.txt.gz') + >>> fp.read() + >>> fp.close() + >>> del ds, fp + +""" + __docformat__ = "restructuredtext en" -import os -import gzip import bz2 -from urlparse import urlparse +import gzip +import os +import tempfile +from shutil import rmtree from urllib2 import urlopen, URLError -from tempfile import mkstemp +from urlparse import urlparse import warnings @@ -31,367 +45,252 @@ warnings.warn(_api_warning) -# TODO: Don't check file extensions, just try and open zip files? -# Follow the, it's easier to get forgiveness than permission? -# And because this shouldn't succeed: -# In [134]: datasource._iszip('/my/fake/dir/foobar.gz') -# Out[134]: True - # TODO: .zip support, .tar support? _file_openers = {".gz":gzip.open, ".bz2":bz2.BZ2File, None:file} -def _iszip(filename): - """Test if the given file is a zip file. - Currently only looks at the file extension, not very robust. - """ +class DataSource (object): + """A generic data source (file, http, ftp, ...). - fname, ext = os.path.splitext(filename) - return ext in _file_openers.keys() + DataSource could be from a local file or remote file/URL. The file may also + be compressed or uncompressed. -def _unzip(filename): - """Unzip the given file and return the path object to the new file.""" + Ex URL DataSources: + Initialize DataSource with a local directory. Default is os.curdir - # This function is not used in datasource. Appears it was created - # so users could unzip a file without having to import the corresponding - # compression module. Should this be part of datasource? - if not _iszip(filename): - raise ValueError("file %s is not zipped"%filename) - unzip_name, zipext = _splitzipext(filename) - opener = _file_openers[zipext] - outfile = file(unzip_name, 'w') - outfile.write(opener(filename).read()) - outfile.close() - return unzip_name + >>> ds = DataSource('/home/guido') + >>> ds.open('http://fake.xyz.web/site/xyz.txt') -def _iswritemode(mode): - """Test if the given mode will open a file for writing.""" + Opened file exists here: /home/guido/site/xyz.txt - _writemodes = ("w", "+", "a") - for c in mode: - if c in _writemodes: return True - return False + Ex using DataSource for temporary files: + Initialize DataSource with 'None' for local directory. -def _splitzipext(filename): - """Split the filename into a path object and a zip extension. + >>> ds = DataSource(None) + >>> ds.open('/home/guido/foobar.txt') - If the filename does not have a zip extension the zip_ext in the - return will be None. + Opened file exists in tempdir like: /tmp/tmpUnhcvM/foobar.txt - Parameters: - - filename : {string} - Filename to split. - - Returns: - - base, zip_ext : {tuple} - Tuple containing a path object to the file and the zip extension. - """ - if _iszip(filename): - return os.path.splitext(filename) - else: - return filename, None + def __init__(self, destpath=os.curdir): + if destpath: + self._destpath = destpath + self._istmpdest = False + else: + self._destpath = tempfile.mkdtemp() + self._istmpdest = True -def _isurl(pathstr): - """Test whether a given string can be parsed as a URL. + def __del__(self): + if self._istmpdest: + rmtree(self._destpath) - A pathstr with a valid network scheme (http, ftp, ...) will return true. - A pathstr with a 'file' scheme will return false. + def _iszip(self, filename): + """Test if the filename is a zip file by looking at the file extension. + """ + fname, ext = os.path.splitext(filename) + return ext in _file_openers.keys() - """ + def _iswritemode(self, mode): + """Test if the given mode will open a file for writing.""" - scheme, netloc, upath, uparams, uquery, ufrag = urlparse(pathstr) - return bool(scheme and netloc) + # Currently only used to test the bz2 files. Not thoroughly tested! + _writemodes = ("w", "+") + for c in mode: + if c in _writemodes: return True + return False -def _ensuredirs(directory): - """Ensure that the given directory path exists. If not, create it.""" + def _splitzipext(self, filename): + """Split zip extension from filename and return filename. - if not os.path.exists(directory): - os.makedirs(directory) - -class _Cache (object): - """A local file cache for URL datasources. - - The path of the cache can be specified on intialization. The default - path is ~/.scipy/cache - - - """ - - def __init__(self, cachepath=None): - if cachepath is not None: - self.path = cachepath - elif os.name == 'posix': - self.path = os.path.join(os.environ["HOME"], ".scipy", "cache") - elif os.name == 'nt': - self.path = os.path.join(os.environ["HOMEPATH"], ".scipy", "cache") - if not os.path.exists(self.path): - _ensuredirs(self.path) - - def tempfile(self, suffix='', prefix=''): - """Create and return a temporary file in the cache. - - Parameters: - suffix : {''}, optional - - prefix : {''}, optional - Returns: - tmpfile : {string} - String containing the full path to the temporary file. + base, zip_ext : {tuple} - Examples - - >>> mycache = datasource._Cache() - >>> mytmpfile = mycache.tempfile() - >>> mytmpfile - '/home/guido/.scipy/cache/GUPhDv' - """ - _tmp, fname = mkstemp(suffix, prefix, self.path) - return fname + if self._iszip(filename): + return os.path.splitext(filename) + else: + return filename, None - def filepath(self, uri): - """Return a path object to the uri in the cache. + def _possible_names(self, filename): + """Return a tuple containing compressed filename variations.""" + names = [filename] + if not self._iszip(filename): + for zipext in _file_openers.keys(): + if zipext: + names.append(filename+zipext) + return names - Parameters: - uri : {string} - Filename to use in the returned path object. + def _isurl(self, path): + """Test if path is a net location. Tests the scheme and netloc.""" + scheme, netloc, upath, uparams, uquery, ufrag = urlparse(path) + return bool(scheme and netloc) - Returns: - path : {string} - Complete path for the given uri. + def _cache(self, path): + """Cache the file specified by path. - Examples + Creates a copy of the file in the datasource cache. - >>> mycache = datasource._Cache() - >>> mycache.filepath('xyzcoords.txt') - '/home/guido/.scipy/cache/xyzcoords.txt' - """ - scheme, netloc, upath, uparams, uquery, ufrag = urlparse(uri) - return os.path.join(self.path, netloc, upath.strip('/')) + upath = self.abspath(path) - def cache(self, uri): - """Copy the file at uri into the cache. + # ensure directory exists + if not os.path.exists(os.path.dirname(upath)): + os.makedirs(os.path.dirname(upath)) - Parameters: - uri : {string} - path or url of source file to cache. - - Returns: - None - - """ - - if self.iscached(uri): - return - - upath = self.filepath(uri) - _ensuredirs(os.path.dirname(upath)) - - print 'cache - source:', uri - print 'cache - destination:', upath - - if _isurl(uri): + # TODO: Doesn't handle compressed files! + if self._isurl(path): try: - openedurl = urlopen(uri) + openedurl = urlopen(path) file(upath, 'w').write(openedurl.read()) except URLError: - raise URLError("URL not found: " + str(uri)) + raise URLError("URL not found: ", path) else: try: - fp = file(uri, 'r') + # TODO: Why not just copy the file with shutils.copyfile? + fp = file(path, 'r') file(upath, 'w').write(fp.read()) except IOError: - raise IOError("File not founcd: " + str(uri)) + raise IOError("File not found: ", path) + return upath - def clear(self): - """Delete all files in the cache.""" + def _findfile(self, path): + """Searches for path and returns full path if found. - # TODO: This deletes all files in the cache directory, regardless - # of if this instance created them. Too destructive and - # unexpected behavior. - #for _file in self.path.files(): - # os.remove(file) - raise NotImplementedError + If path is an URL, _findfile will cache a local copy and return + the path to the cached file. + If path is a local file, _findfile will return a path to that local + file. - def iscached(self, uri): - """ Check if a file exists in the cache. + The search will include possible compressed versions of the file and + return the first occurence found. - Returns - boolean - """ - upath = self.filepath(uri) - return os.path.exists(upath) + # Build list of possible local file paths + filelist = self._possible_names(self.abspath(path)) + if self._isurl(path): + # Add list of possible remote urls + filelist = filelist + self._possible_names(path) - def retrieve(self, uri): - """Retrieve a file from the cache. - If not already there, create the file and add it to the cache. + for name in filelist: + if self.exists(name): + if self._isurl(name): + name = self._cache(name) + return name + return None - Returns - open file object + def abspath(self, path): + """Return an absolute path in the DataSource destination directory. """ - self.cache(uri) - return file(self.filepath(uri)) + # handle case where path includes self._destpath + splitpath = path.split(self._destpath, 2) + if len(splitpath) > 1: + path = splitpath[1] + scheme, netloc, upath, uparams, uquery, ufrag = urlparse(path) + return os.path.join(self._destpath, netloc, upath.strip(os.sep)) + def exists(self, path): + """Test if path exists. -class DataSource (object): - """A generic data source class. + Tests for local files, locally cached URLs and remote URLs. - Data sets could be from a file, a URL, or a cached file. They may also - be compressed or uncompressed. - - TODO: Improve DataSource docstring - - """ - - def __init__(self, cachepath=os.curdir): - self._cache = _Cache(cachepath) - - def tempfile(self, suffix='', prefix=''): - """Create a temporary file in the DataSource cache. - - Parameters: - suffix : {''}, optional - - prefix : {''}, optional - - Returns: - tmpfile : {string} - String containing the full path to the temporary file. - - Examples - - >>> datasrc = datasource.DataSource() - >>> tmpfile = datasrc.tempfile() - >>> tmpfile - '/home/guido/src/scipy-trunk/scipy/io/PZTuKo' - """ - return self._cache.tempfile(suffix, prefix) - def _possible_names(self, filename): - """Return a tuple containing compressed filenames.""" - names = [filename] - if not _iszip(filename): - for zipext in _file_openers.keys(): - names.append(filename+zipext) - return tuple(names) - - def cache(self, pathstr): - """Cache the file specified by pathstr. - - Creates a copy of file pathstr in the datasource cache. - - """ - - self._cache.cache(pathstr) - - def clear(self): - # TODO: Implement a clear interface for deleting tempfiles. - # There's a problem with the way this is handled in the _Cache, - # All files in the cache directory will be deleted. In the - # default instance, that's the os.curdir. I doubt this is what - # people would want. The instance should only delete files that - # it created! - raise NotImplementedError - - def filename(self, pathstr): - """Searches for pathstr file and returns full path if found. - - If pathstr is an URL, filename will cache a local copy and return - the path to the cached file. - If pathstr is a local file, filename will return a path to that local - file. - BUG: This should be modified so the behavior is identical for both - types of files! - - The search will include possible compressed versions of the files. - BUG: Will return the first occurence found, regardless of which - version is newer. - - """ - - found = None - for name in self._possible_names(pathstr): - try: - if _isurl(name): - self.cache(name) - found = self._cache.filepath(name) - else: - raise Exception - except: - if os.path.exists(name): - found = name - if found: - break - if found is None: - raise IOError("%s not found"%pathstr) - return found - - def exists(self, pathstr): - """Test if pathstr exists in the cache or the current directory. - - If pathstr is an URL, it will be fetched and cached. - - """ - - # Is this method doing to much? At very least may want an option to - # not fetch and cache URLs. - - try: - _datafile = self.filename(pathstr) + upath = self.abspath(path) + if os.path.exists(upath): return True - except IOError: + elif self._isurl(path): + try: + netfile = urlopen(path) + # just validate existence, nothing more. + del(netfile) + return True + except URLError: + return False + else: return False - def open(self, pathstr, mode='r'): - """Open pathstr and return file object. + def open(self, path, mode='r'): + """Open path and return file object. - If pathstr is an URL, it will be fetched and cached. + If path is an URL, it will be downloaded, stored in the DataSource + directory and opened. + TODO: Currently only opening for reading has been tested. There is no + support for opening a file for writing which doesn't exist yet + (creating a file). + """ - # Is this method doing to much? Should it be fetching and caching? - - if _isurl(pathstr) and _iswritemode(mode): + if self._isurl(path) and self._iswritemode(mode): raise ValueError("URLs are not writeable") - found = self.filename(pathstr) - _fname, ext = _splitzipext(found) - if ext == 'bz2': - mode.replace("+", "") - return _file_openers[ext](found, mode=mode) + # NOTE: _findfile will fail on a new file opened for writing. + found = self._findfile(path) + if found: + _fname, ext = self._splitzipext(found) + if ext == 'bz2': + mode.replace("+", "") + return _file_openers[ext](found, mode=mode) + else: + raise IOError("%s not found." % path) + class Repository (DataSource): - """Multiple DataSource's that share one base url. + """A data repository where multiple DataSource's share one base URL. - TODO: Improve Repository docstring. + Use a Repository when you will be working with multiple files from one + base URL or directory. Initialize the Respository with the base URL, + then refer to each file only by it's filename. + >>> repos = Repository('/home/user/data/dir/') + >>> fp = repos.open('data01.txt') + >>> fp.analyze() + >>> fp.close() + + Similarly you could use a URL for a repository: + >>> repos = Repository('http://www.xyz.edu/data') + """ - def __init__(self, baseurl, cachepath=None): - DataSource.__init__(self, cachepath=cachepath) + def __init__(self, baseurl, destpath=os.curdir): + DataSource.__init__(self, destpath=destpath) self._baseurl = baseurl - def _fullpath(self, pathstr): - return os.path.join(self._baseurl, pathstr) + def _fullpath(self, path): + '''Return complete path for path. Prepends baseurl if necessary.''' + #print 'Repository._fullpath:', path + #print ' ._baseurl: ', self._baseurl + splitpath = path.split(self._baseurl, 2) + if len(splitpath) == 1: + result = os.path.join(self._baseurl, path) + else: + result = path # path contains baseurl already + return result - def filename(self, pathstr): - return DataSource.filename(self, self._fullpath(pathstr)) + def _findfile(self, path): + #print 'Repository._findfile:', path + return DataSource._findfile(self, self._fullpath(path)) - def exists(self, pathstr): - return DataSource.exists(self, self._fullpath(pathstr)) + def abspath(self, path): + return DataSource.abspath(self, self._fullpath(path)) - def open(self, pathstr, mode='r'): - return DataSource.open(self, self._fullpath(pathstr), mode) + def exists(self, path): + #print 'Respository.exists:', path + return DataSource.exists(self, self._fullpath(path)) + + def open(self, path, mode='r'): + #print 'Repository.open:', path + return DataSource.open(self, self._fullpath(path), mode) + + def listdir(self): + '''List files in the source Repository.''' + if self._isurl(self._baseurl): + raise NotImplementedError + else: + return os.listdir(self._baseurl) Added: trunk/scipy/io/tests/test_datasource.py =================================================================== --- trunk/scipy/io/tests/test_datasource.py 2007-10-30 22:43:58 UTC (rev 3474) +++ trunk/scipy/io/tests/test_datasource.py 2007-10-31 18:02:38 UTC (rev 3475) @@ -0,0 +1,207 @@ + +import bz2 +import gzip +import os +import sys +import struct +from tempfile import mkdtemp, mkstemp +from shutil import rmtree +from urlparse import urlparse + +from numpy.testing import * + +# HACK: import the src datasource +# Until datasource is robust enough to include in scipy.io package +sys.path.insert(0, os.path.abspath('..')) +import datasource +del sys.path[0] + +#set_package_path() +#from scipy.io import datasource +#restore_path() + +# Can rebind urlopen for testing, so we don't open real net connection. +#def urlopen(url, data=None): +# print 'test_datasource urlopen(', url, ')' +#datasource.urlopen = urlopen + +#http_baseurl = 'http://nifti.nimh.nih.gov/nifti-1/data/' +#http_filename = 'minimal.nii.gz' +#http_abspath = os.path.join(http_baseurl, http_filename) + +# Temporarily use one of our files so we don't abuse someone elses server. +http_path = 'https://cirl.berkeley.edu/twiki/pub/BIC/ImagingDocuments/' +http_file = 'dork.pdf' + +http_fakepath = 'http://fake.abc.web/site/' +http_fakefile = 'fake.txt' + +magic_line = 'three is the magic number' + + +# Utility functions used by many TestCases +def valid_textfile(filedir): + # Generate and return a valid temporary file. + fd, path = mkstemp(suffix='.txt', dir=filedir, text=True) + os.close(fd) + return path + +def invalid_textfile(filedir): + # Generate and return an invalid filename. + fd, path = mkstemp(suffix='.txt', dir=filedir) + os.close(fd) + os.remove(path) + return path + +def valid_httpurl(): + return http_path+http_file + +def invalid_httpurl(): + return http_fakepath+http_fakefile + +def valid_baseurl(): + return http_path + +def invalid_baseurl(): + return http_fakepath + +def valid_httpfile(): + return http_file + +def invalid_httpfile(): + return http_fakefile + +class TestDataSourceOpen(NumpyTestCase): + def setUp(self): + self.tmpdir = mkdtemp() + self.ds = datasource.DataSource(self.tmpdir) + + def tearDown(self): + rmtree(self.tmpdir) + del self.ds + + def test_ValidHTTP(self): + assert self.ds.open(valid_httpurl()) + + def test_InvalidHTTP(self): + self.assertRaises(IOError, self.ds.open, invalid_httpurl()) + + def test_ValidFile(self): + local_file = valid_textfile(self.tmpdir) + #print '\nDataSourceOpen test_ValidFile:', local_file + assert self.ds.open(local_file) + + def test_InvalidFile(self): + invalid_file = invalid_textfile(self.tmpdir) + #print '\nDataSourceOpen test_InvalidFile:', invalid_file + self.assertRaises(IOError, self.ds.open, invalid_file) + + def test_ValidGzipFile(self): + # Test datasource's internal file_opener for Gzip files. + filepath = os.path.join(self.tmpdir, 'foobar.txt.gz') + #print '\nDataSourceOpen test_ValidGzipFile:', filepath + fp = gzip.open(filepath, 'w') + fp.write(magic_line) + fp.close() + fp = self.ds.open(filepath) + result = fp.readline() + fp.close() + self.assertEqual(magic_line, result) + + def test_ValidBz2File(self): + # Test datasource's internal file_opener for BZip2 files. + filepath = os.path.join(self.tmpdir, 'foobar.txt.bz2') + #print '\nDataSourceOpen test_ValidBZ2File:', filepath + fp = bz2.BZ2File(filepath, 'w') + fp.write(magic_line) + fp.close() + fp = self.ds.open(filepath) + result = fp.readline() + fp.close() + self.assertEqual(magic_line, result) + + +class TestDataSourceExists(NumpyTestCase): + def setUp(self): + self.tmpdir = mkdtemp() + self.ds = datasource.DataSource(self.tmpdir) + + def tearDown(self): + rmtree(self.tmpdir) + del self.ds + + def test_ValidHTTP(self): + #print 'DataSourceExists test_ValidHTTP' + assert self.ds.exists(valid_httpurl()) + + def test_InvalidHTTP(self): + #print 'DataSourceExists test_InvalidHTTP' + self.assertEqual(self.ds.exists(invalid_httpurl()), False) + + def test_ValidFile(self): + tmpfile = valid_textfile(self.tmpdir) + #print 'DataSourceExists test_ValidFile:', tmpfile + assert self.ds.exists(tmpfile) + + def test_InvalidFile(self): + tmpfile = invalid_textfile(self.tmpdir) + #print 'DataSourceExists test_InvalidFile:', tmpfile + self.assertEqual(self.ds.exists(tmpfile), False) + + +class TestDataSourceAbspath(NumpyTestCase): + def setUp(self): + self.tmpdir = mkdtemp() + self.ds = datasource.DataSource(self.tmpdir) + + def tearDown(self): + rmtree(self.tmpdir) + del self.ds + + def test_ValidHTTP(self): + scheme, netloc, upath, pms, qry, frg = urlparse(valid_httpurl()) + local_path = os.path.join(self.tmpdir, netloc, upath.strip(os.sep)) + self.assertEqual(local_path, self.ds.abspath(valid_httpurl())) + + def test_ValidFile(self): + tmpfile = valid_textfile(self.tmpdir) + tmpfilename = os.path.split(tmpfile)[-1] + # Test with filename only + self.assertEqual(tmpfile, self.ds.abspath(os.path.split(tmpfile)[-1])) + # Test filename with complete path + self.assertEqual(tmpfile, self.ds.abspath(tmpfile)) + + def test_InvalidHTTP(self): + scheme, netloc, upath, pms, qry, frg = urlparse(invalid_httpurl()) + invalidhttp = os.path.join(self.tmpdir, netloc, upath.strip(os.sep)) + self.assertNotEqual(invalidhttp, self.ds.abspath(valid_httpurl())) + + def test_InvalidFile(self): + fd, invalidfile = mkstemp(suffix='.txt') + os.close(fd) + tmpfile = valid_textfile(self.tmpdir) + tmpfilename = os.path.split(tmpfile)[-1] + # Test with filename only + self.assertNotEqual(invalidfile, self.ds.abspath(tmpfilename)) + # Test filename with complete path + self.assertNotEqual(invalidfile, self.ds.abspath(tmpfile)) + + +class TestRespositoryAbspath(NumpyTestCase): + def setUp(self): + self.repos = datasource.Repository(valid_baseurl(), None) + + def tearDown(self): + del self.repos + + def test_ValidHTTP(self): + scheme, netloc, upath, pms, qry, frg = urlparse(valid_httpurl()) + local_path = os.path.join(self.repos._destpath, netloc, \ + upath.strip(os.sep)) + filepath = self.repos.abspath(valid_httpfile()) + self.assertEqual(local_path, filepath) + + +if __name__ == "__main__": + NumpyTest().run() + From scipy-svn at scipy.org Wed Oct 31 16:35:12 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 31 Oct 2007 15:35:12 -0500 (CDT) Subject: [Scipy-svn] r3476 - trunk/scipy/io/tests Message-ID: <20071031203512.20D6539C280@new.scipy.org> Author: chris.burns Date: 2007-10-31 15:35:09 -0500 (Wed, 31 Oct 2007) New Revision: 3476 Modified: trunk/scipy/io/tests/test_datasource.py Log: Cleanup test_datasource, add urlopen_stub for testing. Modified: trunk/scipy/io/tests/test_datasource.py =================================================================== --- trunk/scipy/io/tests/test_datasource.py 2007-10-31 18:02:38 UTC (rev 3475) +++ trunk/scipy/io/tests/test_datasource.py 2007-10-31 20:35:09 UTC (rev 3476) @@ -4,7 +4,7 @@ import os import sys import struct -from tempfile import mkdtemp, mkstemp +from tempfile import mkdtemp, mkstemp, NamedTemporaryFile from shutil import rmtree from urlparse import urlparse @@ -20,18 +20,21 @@ #from scipy.io import datasource #restore_path() -# Can rebind urlopen for testing, so we don't open real net connection. -#def urlopen(url, data=None): -# print 'test_datasource urlopen(', url, ')' -#datasource.urlopen = urlopen +def urlopen_stub(url, data=None): + '''Stub to replace urlopen for testing.''' + if url == valid_httpurl(): + tmpfile = NamedTemporaryFile(prefix='urltmp_') + return tmpfile + else: + raise datasource.URLError('Name or service not known') -#http_baseurl = 'http://nifti.nimh.nih.gov/nifti-1/data/' -#http_filename = 'minimal.nii.gz' -#http_abspath = os.path.join(http_baseurl, http_filename) +# Rebind urlopen during testing. For a 'real' test, uncomment the rebinding +# below. +datasource.urlopen = urlopen_stub -# Temporarily use one of our files so we don't abuse someone elses server. -http_path = 'https://cirl.berkeley.edu/twiki/pub/BIC/ImagingDocuments/' -http_file = 'dork.pdf' +# A valid website for more robust testing +http_path = 'http://www.google.com/' +http_file = 'index.html' http_fakepath = 'http://fake.abc.web/site/' http_fakefile = 'fake.txt' @@ -42,13 +45,13 @@ # Utility functions used by many TestCases def valid_textfile(filedir): # Generate and return a valid temporary file. - fd, path = mkstemp(suffix='.txt', dir=filedir, text=True) + fd, path = mkstemp(suffix='.txt', prefix='dstmp_', dir=filedir, text=True) os.close(fd) return path def invalid_textfile(filedir): # Generate and return an invalid filename. - fd, path = mkstemp(suffix='.txt', dir=filedir) + fd, path = mkstemp(suffix='.txt', prefix='dstmp_', dir=filedir) os.close(fd) os.remove(path) return path @@ -88,18 +91,15 @@ def test_ValidFile(self): local_file = valid_textfile(self.tmpdir) - #print '\nDataSourceOpen test_ValidFile:', local_file assert self.ds.open(local_file) def test_InvalidFile(self): invalid_file = invalid_textfile(self.tmpdir) - #print '\nDataSourceOpen test_InvalidFile:', invalid_file self.assertRaises(IOError, self.ds.open, invalid_file) def test_ValidGzipFile(self): # Test datasource's internal file_opener for Gzip files. filepath = os.path.join(self.tmpdir, 'foobar.txt.gz') - #print '\nDataSourceOpen test_ValidGzipFile:', filepath fp = gzip.open(filepath, 'w') fp.write(magic_line) fp.close() @@ -111,7 +111,6 @@ def test_ValidBz2File(self): # Test datasource's internal file_opener for BZip2 files. filepath = os.path.join(self.tmpdir, 'foobar.txt.bz2') - #print '\nDataSourceOpen test_ValidBZ2File:', filepath fp = bz2.BZ2File(filepath, 'w') fp.write(magic_line) fp.close() @@ -131,21 +130,17 @@ del self.ds def test_ValidHTTP(self): - #print 'DataSourceExists test_ValidHTTP' assert self.ds.exists(valid_httpurl()) def test_InvalidHTTP(self): - #print 'DataSourceExists test_InvalidHTTP' self.assertEqual(self.ds.exists(invalid_httpurl()), False) def test_ValidFile(self): tmpfile = valid_textfile(self.tmpdir) - #print 'DataSourceExists test_ValidFile:', tmpfile assert self.ds.exists(tmpfile) def test_InvalidFile(self): tmpfile = invalid_textfile(self.tmpdir) - #print 'DataSourceExists test_InvalidFile:', tmpfile self.assertEqual(self.ds.exists(tmpfile), False) @@ -177,8 +172,7 @@ self.assertNotEqual(invalidhttp, self.ds.abspath(valid_httpurl())) def test_InvalidFile(self): - fd, invalidfile = mkstemp(suffix='.txt') - os.close(fd) + invalidfile = valid_textfile(self.tmpdir) tmpfile = valid_textfile(self.tmpdir) tmpfilename = os.path.split(tmpfile)[-1] # Test with filename only From scipy-svn at scipy.org Wed Oct 31 17:46:31 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 31 Oct 2007 16:46:31 -0500 (CDT) Subject: [Scipy-svn] r3477 - trunk/scipy/io Message-ID: <20071031214631.A05C839C05B@new.scipy.org> Author: chris.burns Date: 2007-10-31 16:46:29 -0500 (Wed, 31 Oct 2007) New Revision: 3477 Modified: trunk/scipy/io/datasource.py Log: Update documentation and reST format. Modified: trunk/scipy/io/datasource.py =================================================================== --- trunk/scipy/io/datasource.py 2007-10-31 20:35:09 UTC (rev 3476) +++ trunk/scipy/io/datasource.py 2007-10-31 21:46:29 UTC (rev 3477) @@ -1,20 +1,29 @@ -"""A generic file interface for handling data files. The goal of datasource is -to abstract some of the file system operations when dealing with files. -Specifically acquiring the files. DataSource files can originate locally or -remotely, for example: - local files - '/home/name/blah/blah/blah/data.txt' - URLs (http, ftp, ...) - 'http://www.scipy.org/not/real/data.txt' +"""A file interface for handling local and remote data files. +The goal of datasource is to abstract some of the file system operations when +dealing with data files so the researcher doesn't have to know all the +low-level details. Through datasource, a researcher can obtain and use a +file with one function call, regardless of location of the file. +DataSource files can originate locally or remotely: + +- local files : '/home/guido/src/local/data.txt' +- URLs (http, ftp, ...) : 'http://www.scipy.org/not/real/data.txt' + DataSource files can also be compressed or uncompressed. Currently only gzip and bz2 are supported. -In a typical use, you would pass a DataSource to a function that would open -up the DataSource (as it would any file-like object) and read the file. +Example: -Ex: - >>> ds = datasource.DataSource() + >>> # Create a DataSource and use '/home/guido/tmpdata/' for local storage. + >>> ds = datasource.DataSource('/home/guido/tmpdata/') + >>> + >>> # Open a remote, gzipped file. + >>> # DataSource downloads the file, stores it locally in: + >>> # '/home/guido/tmpdata/www.scipy.org/not/real/data.txt.gz' + >>> # opens the file with the gzip module and returns a file-like object. + >>> >>> fp = ds.open('http://www.scipy.org/not/real/data.txt.gz') - >>> fp.read() + >>> fp.read() # Use the file >>> fp.close() >>> del ds, fp @@ -52,8 +61,8 @@ class DataSource (object): """A generic data source (file, http, ftp, ...). - DataSource could be from a local file or remote file/URL. The file may also - be compressed or uncompressed. + DataSource could be from a local file or remote file/URL. The file may + also be compressed or uncompressed. Ex URL DataSources: Initialize DataSource with a local directory. Default is os.curdir @@ -97,7 +106,8 @@ # Currently only used to test the bz2 files. Not thoroughly tested! _writemodes = ("w", "+") for c in mode: - if c in _writemodes: return True + if c in _writemodes: + return True return False def _splitzipext(self, filename): From scipy-svn at scipy.org Wed Oct 31 21:31:40 2007 From: scipy-svn at scipy.org (scipy-svn at scipy.org) Date: Wed, 31 Oct 2007 20:31:40 -0500 (CDT) Subject: [Scipy-svn] r3478 - trunk/scipy/sandbox/multigrid Message-ID: <20071101013140.69041C7C02E@new.scipy.org> Author: wnbell Date: 2007-10-31 20:31:37 -0500 (Wed, 31 Oct 2007) New Revision: 3478 Modified: trunk/scipy/sandbox/multigrid/adaptive.py trunk/scipy/sandbox/multigrid/dec_test.py trunk/scipy/sandbox/multigrid/multilevel.py trunk/scipy/sandbox/multigrid/sa.py Log: added new method for curl-curl problem Modified: trunk/scipy/sandbox/multigrid/adaptive.py =================================================================== --- trunk/scipy/sandbox/multigrid/adaptive.py 2007-10-31 21:46:29 UTC (rev 3477) +++ trunk/scipy/sandbox/multigrid/adaptive.py 2007-11-01 01:31:37 UTC (rev 3478) @@ -1,18 +1,18 @@ import numpy,scipy,scipy.sparse from numpy import sqrt, ravel, diff, zeros, zeros_like, inner, concatenate, \ - asarray, hstack, ascontiguousarray, isinf + asarray, hstack, ascontiguousarray, isinf, dot from numpy.random import randn from scipy.sparse import csr_matrix,coo_matrix from relaxation import gauss_seidel from multilevel import multilevel_solver from sa import sa_constant_interpolation,sa_fit_candidates -from utils import approximate_spectral_radius,hstack_csr,vstack_csr,expand_into_blocks +from utils import approximate_spectral_radius,hstack_csr,vstack_csr,expand_into_blocks,diag_sparse def augment_candidates(AggOp, old_Q, old_R, new_candidate): - #TODO update P also + #TODO update P and A also K = old_R.shape[1] @@ -124,7 +124,7 @@ class adaptive_sa_solver: - def __init__(self, A, blocks=None, max_levels=10, max_coarse=100,\ + def __init__(self, A, blocks=None, aggregation=None, max_levels=10, max_coarse=100,\ max_candidates=1, mu=5, epsilon=0.1): self.A = A @@ -138,8 +138,9 @@ x,AggOps = self.__initialization_stage(A, blocks = blocks, \ max_levels = max_levels, \ max_coarse = max_coarse, \ - mu = mu, epsilon = epsilon) - + mu = mu, epsilon = epsilon, \ + aggregation = aggregation ) + #create SA using x here As,Ps,Ts,Bs = sa_hierarchy(A,x,AggOps) @@ -147,8 +148,20 @@ x = self.__develop_new_candidate(As,Ps,Ts,Bs,AggOps,mu=mu) #TODO which is faster? - #As,Ps,Ts,Bs = self.__augment_cycle(As,Ps,Ts,Bs,AggOps,x) - B = hstack((Bs[0],x)) + As,Ps,Ts,Bs = self.__augment_cycle(As,Ps,Ts,Bs,AggOps,x) + + #B = hstack((Bs[0],x)) + #As,Ps,Ts,Bs = sa_hierarchy(A,B,AggOps) + + #improve candidates? + if True: + print "improving candidates" + B = Bs[0] + for i in range(max_candidates): + B = B[:,1:] + As,Ps,Ts,Bs = sa_hierarchy(A,B,AggOps) + x = self.__develop_new_candidate(As,Ps,Ts,Bs,AggOps,mu=mu) + B = hstack((B,x)) As,Ps,Ts,Bs = sa_hierarchy(A,B,AggOps) self.Ts = Ts @@ -156,12 +169,12 @@ self.AggOps = AggOps self.Bs = Bs + def __initialization_stage(self,A,blocks,max_levels,max_coarse,mu,epsilon,aggregation): + if aggregation is not None: + max_coarse = 0 + max_levels = len(aggregation) + 1 - def __initialization_stage(self,A,blocks,max_levels,max_coarse,mu,epsilon): - AggOps = [] - Ps = [] - - # aSA parameters + # aSA parameters # mu - number of test relaxation iterations # epsilon - minimum acceptable relaxation convergence factor @@ -177,9 +190,14 @@ #TODO test convergence rate here As = [A] + AggOps = [] + Ps = [] - while len(AggOps) + 1 < max_levels and A_l.shape[0] > max_coarse: - W_l = sa_constant_interpolation(A_l,epsilon=0,blocks=blocks) #step 4b + while len(AggOps) + 1 < max_levels and A_l.shape[0] > max_coarse: + if aggregation is None: + W_l = sa_constant_interpolation(A_l,epsilon=0,blocks=blocks) #step 4b + else: + W_l = aggregation[len(AggOps)] P_l,x = sa_fit_candidates(W_l,x) #step 4c I_l = smoothed_prolongator(P_l,A_l) #step 4d A_l = I_l.T.tocsr() * A_l * I_l #step 4e @@ -286,8 +304,11 @@ from multilevel import poisson_problem1D,poisson_problem2D blocks = None + aggregation = None - A = poisson_problem2D(100) + #A = poisson_problem2D(200,1e-2) + #aggregation = [ sa_constant_interpolation(A*A*A,epsilon=0.0) ] + #A = io.mmread("tests/sample_data/laplacian_41_3dcube.mtx").tocsr() #A = io.mmread("laplacian_40_3dcube.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() @@ -301,17 +322,17 @@ blocks = arange(A.shape[0]/2).repeat(2) from time import clock; start = clock() - asa = adaptive_sa_solver(A,max_candidates=5,mu=6,blocks=blocks) + asa = adaptive_sa_solver(A,max_candidates=3,mu=5,blocks=blocks,aggregation=aggregation) print "Adaptive Solver Construction: %s seconds" % (clock() - start); del start - #scipy.random.seed(0) #make tests repeatable + scipy.random.seed(0) #make tests repeatable x = randn(A.shape[0]) b = A*randn(A.shape[0]) #b = zeros(A.shape[0]) print "solving" - if True: + if False: x_sol,residuals = asa.solver.solve(b,x0=x,maxiter=20,tol=1e-12,return_residuals=True) else: residuals = [] @@ -352,11 +373,12 @@ pcolor(x.reshape(sqrt(len(x)),sqrt(len(x)))) show() + for c in asa.Bs[0].T: + #plot2d(c) plot2d_arrows(c) print "candidate Rayleigh quotient",dot(c,A*c)/dot(c,c) - ##W = asa.AggOps[0]*asa.AggOps[1] ##pcolor((W * rand(W.shape[1])).reshape((200,200))) Modified: trunk/scipy/sandbox/multigrid/dec_test.py =================================================================== --- trunk/scipy/sandbox/multigrid/dec_test.py 2007-10-31 21:46:29 UTC (rev 3477) +++ trunk/scipy/sandbox/multigrid/dec_test.py 2007-11-01 01:31:37 UTC (rev 3478) @@ -1,10 +1,12 @@ from scipy import * +from scipy.sparse import * from pydec import * -from pydec.multigrid import * from pydec.multigrid.discrete_laplacian import boundary_hierarchy, discrete_laplacian_solver, hodge_solver -from scipy.sandbox.multigrid import smoothed_aggregation_solver +from scipy.sandbox.multigrid import smoothed_aggregation_solver,multigridtools,multilevel_solver +from scipy.sandbox.multigrid.adaptive import adaptive_sa_solver +from scipy.sandbox.multigrid.sa import sa_smoothed_prolongator from scipy.sandbox.multigrid.utils import expand_into_blocks @@ -13,8 +15,9 @@ #mesh = read_mesh(mesh_path + 'rocket/rocket.xml') #mesh = read_mesh(mesh_path + 'genus3/genus3_168k.xml') #mesh = read_mesh(mesh_path + 'genus3/genus3_455k.xml') -mesh = read_mesh(mesh_path + '/torus/torus.xml') -for i in range(3): +#mesh = read_mesh(mesh_path + '/torus/torus.xml') +mesh = read_mesh(mesh_path + '/sq14tri/sq14tri.xml') +for i in range(5): mesh['vertices'],mesh['elements'] = loop_subdivision(mesh['vertices'],mesh['elements']) cmplx = simplicial_complex(mesh['vertices'],mesh['elements']) @@ -24,8 +27,34 @@ #bitmap[100:150,100:400] = False #cmplx = regular_cube_complex(regular_cube_mesh(bitmap)) +def curl_curl_prolongator(D_nodal,vertices): + if not isspmatrix_csr(D_nodal): + raise TypeError('expected csr_matrix') + + A = D_nodal.T.tocsr() * D_nodal + aggs = multigridtools.sa_get_aggregates(A.shape[0],A.indptr,A.indices) + + num_edges = D_nodal.shape[0] + num_basis = vertices.shape[1] + num_aggs = aggs.max() + 1 + # replace with CSR + eliminate duplicates + #indptr = (2*num_basis) * arange(num_edges+1) + ## same same + #csr_matrix((data,indices,indptr),dims=(num_edges,num_aggs)) + row = arange(num_edges).repeat(2*num_basis) + col = (num_basis*aggs[D_nodal.indices]).repeat(num_basis) + col = col.reshape(-1,num_basis) + arange(num_basis) + col = col.reshape(-1) + data = tile(0.5 * (D_nodal*vertices),(1,2)).reshape(-1) + + return coo_matrix((data,(row,col)),dims=(num_edges,num_basis*num_aggs)).tocsr() + + + + + def whitney_innerproduct_cache(cmplx,k): h = hash(cmplx.vertices.tostring()) ^ hash(cmplx.simplices.tostring()) ^ hash(k) @@ -73,47 +102,83 @@ Mi = whitney_innerproduct_cache(cmplx,i+1) else: Mi = regular_cube_innerproduct(cmplx,i+1) + + + dimension = mesh['vertices'].shape[1] - ##print "constructing solver" - ##ss = discrete_laplacian_solver(cochain_complex,len(cochain_complex)-i-1,innerproduct=Mi) - ##print ss - ## - ##print "solving" - ##x,res = ss.solve(b=zeros(ss.A.shape[0]),x0=rand(ss.A.shape[0]),return_residuals=True) - - bh = boundary_hierarchy(cochain_complex) - while len(bh) < 3: - bh.coarsen() - print repr(bh) - - N = len(cochain_complex) - 1 - - B = bh[0][N - i].B - - A = B.T.tocsr() * B - #A = B.T.tocsr() * Mi * B - - constant_prolongators = [lvl[N - i].I for lvl in bh[:-1]] - - if i == 0: + if True: + + d0 = cmplx[0].d + d1 = cmplx[1].d + + #A = (d1.T.tocsr() * d1 + d0 * d0.T.tocsr()).astype('d') + A = (d1.T.tocsr() * d1).astype('d') + + P = curl_curl_prolongator(d0,mesh['vertices']) + + num_blocks = P.shape[1]/dimension + blocks = arange(num_blocks).repeat(dimension) + + P = sa_smoothed_prolongator(A,P,epsilon=0,omega=4.0/3.0) + + PAP = P.T.tocsr() * A * P + candidates = None + candidates = zeros((num_blocks,dimension,dimension)) + for n in range(dimension): + candidates[:,n,n] = 1.0 + candidates = candidates.reshape(-1,dimension) + + ml = smoothed_aggregation_solver(PAP,epsilon=0.0,candidates=candidates,blocks=blocks) + #A = PAP + ml = multilevel_solver([A] + ml.As, [P] + ml.Ps) else: - #candidates = [ones(A.shape[0])] - #TODO test - candidates = [] - for coord in range(mesh['vertices'].shape[1]): - candidates.append( bh[0][N-i+1].B * mesh['vertices'][:,coord] ) + bh = boundary_hierarchy(cochain_complex) + while len(bh) < 3: + bh.coarsen() + print repr(bh) + + N = len(cochain_complex) - 1 + + B = bh[0][N - i].B + + A = (B.T.tocsr() * B).astype('d') + #A = B.T.tocsr() * Mi * B + + constant_prolongators = [lvl[N - i].I for lvl in bh[:-1]] + + method = 'aSA' - K = len(candidates) + if method == 'RS': + As = [A] + Ps = [] + for T in constant_prolongators: + Ps.append( sa_smoothed_prolongator(As[-1],T,epsilon=0.0,omega=4.0/3.0) ) + As.append(Ps[-1].T.tocsr() * As[-1] * Ps[-1]) + ml = multilevel_solver(As,Ps) + + else: + if method == 'BSA': + if i == 0: + candidates = None + else: + candidates = cmplx[0].d * mesh['vertices'] + K = candidates.shape[1] + + constant_prolongators = [constant_prolongators[0]] + \ + [expand_into_blocks(T,K,1).tocsr() for T in constant_prolongators[1:] ] - constant_prolongators = [constant_prolongators[0]] + \ - [expand_into_blocks(T,K,1).tocsr() for T in constant_prolongators[1:] ] + ml = smoothed_aggregation_solver(A,candidates,aggregation=constant_prolongators) + elif method == 'aSA': + asa = adaptive_sa_solver(A,aggregation=constant_prolongators,max_candidates=dimension,epsilon=0.0) + ml = asa.solver + else: + raise ValuerError,'unknown method' + + #ml = smoothed_aggregation_solver(A,candidates) - - ml = smoothed_aggregation_solver(A,candidates,aggregation=constant_prolongators) - #ml = smoothed_aggregation_solver(A,candidates) - + #x = d0 * mesh['vertices'][:,0] x = rand(A.shape[0]) b = zeros_like(x) #b = A*rand(A.shape[0]) @@ -125,9 +190,11 @@ def add_resid(x): residuals.append(linalg.norm(b - A*x)) A.psolve = ml.psolve - x_sol = linalg.cg(A,b,x0=x,maxiter=30,tol=1e-12,callback=add_resid)[0] + from pydec import cg + x_sol = cg(A,b,x0=x,maxiter=40,tol=1e-8,callback=add_resid)[0] + residuals = array(residuals)/residuals[0] avg_convergence_ratio = residuals[-1]**(1.0/len(residuals)) print "average convergence ratio",avg_convergence_ratio Modified: trunk/scipy/sandbox/multigrid/multilevel.py =================================================================== --- trunk/scipy/sandbox/multigrid/multilevel.py 2007-10-31 21:46:29 UTC (rev 3477) +++ trunk/scipy/sandbox/multigrid/multilevel.py 2007-11-01 01:31:37 UTC (rev 3478) @@ -24,14 +24,14 @@ O = -ones(N) return scipy.sparse.spdiags([D,O,O],[0,-1,1],N,N).tocoo().tocsr() #eliminate explicit zeros -def poisson_problem2D(N): +def poisson_problem2D(N,epsilon=1.0): """ Return a sparse CSR matrix for the 2d poisson problem with standard 5-point finite difference stencil on a square N-by-N grid. """ - D = 4*ones(N*N) - T = -ones(N*N) + D = (2 + 2*epsilon)*ones(N*N) + T = -epsilon * ones(N*N) O = -ones(N*N) T[N-1::N] = 0 return scipy.sparse.spdiags([D,O,T,T,O],[0,-N,-1,1,N],N*N,N*N).tocoo().tocsr() #eliminate explicit zeros @@ -208,7 +208,9 @@ #use direct solver on coarsest level #TODO reuse factors for efficiency? coarse_x[:] = spsolve(self.As[-1],coarse_b).reshape(coarse_x.shape) - #coarse_x[:] = scipy.linalg.cg(self.As[-1],coarse_b,tol=1e-12)[0] + #coarse_x[:] = scipy.linalg.cg(self.As[-1],coarse_b,tol=1e-12)[0].reshape(coarse_x.shape) + #A_inv = asarray(scipy.linalg.pinv2(self.As[-1].todense())) + #coarse_x[:] = scipy.dot(A_inv,coarse_b) #print "coarse residual norm",scipy.linalg.norm(coarse_b - self.As[-1]*coarse_x) else: self.__solve(lvl+1,coarse_x,coarse_b) @@ -230,18 +232,17 @@ from scipy import * candidates = None blocks = None - #A = poisson_problem2D(100) + A = poisson_problem2D(40,1e-2) #A = io.mmread("rocker_arm_surface.mtx").tocsr() #A = io.mmread("9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/9pt/9pt-100x100.mtx").tocsr() #A = io.mmread("/home/nathan/Desktop/BasisShift_W_EnergyMin_Luke/9pt-5x5.mtx").tocsr() - A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() - candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') - #candidates = [ array(candidates[:,x]) for x in range(candidates.shape[1]) ] - blocks = arange(A.shape[0]/2).repeat(2) + #A = io.mmread('tests/sample_data/elas30_A.mtx').tocsr() + #candidates = io.mmread('tests/sample_data/elas30_nullspace.mtx') + #blocks = arange(A.shape[0]/2).repeat(2) - ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,epsilon=0,max_coarse=100,max_levels=2) + ml = smoothed_aggregation_solver(A,candidates,blocks=blocks,epsilon=0.08,max_coarse=100,max_levels=10) #ml = ruge_stuben_solver(A) x = rand(A.shape[0]) Modified: trunk/scipy/sandbox/multigrid/sa.py =================================================================== --- trunk/scipy/sandbox/multigrid/sa.py 2007-10-31 21:46:29 UTC (rev 3477) +++ trunk/scipy/sandbox/multigrid/sa.py 2007-11-01 01:31:37 UTC (rev 3478) @@ -182,7 +182,7 @@ P = T - (D_inv_A*T) #S = (spidentity(A.shape[0]).tocsr() - D_inv_A) #TODO drop this? - #P = S * ( S * T) + #P = S *(S * ( S * T)) return P @@ -201,8 +201,6 @@ T,coarse_candidates = sa_fit_candidates(AggOp,candidates) #T = AggOp #TODO test - A_filtered = sa_filtered_matrix(A,epsilon,blocks) #use filtered matrix for anisotropic problems - P = sa_smoothed_prolongator(A,T,epsilon,omega,blocks) if blocks is not None: