[SciPy-dev] scipy.cluster

[Nils Wagner]

David Cournapeau wrote:
> Nils Wagner wrote:
>   
>> Hi all,
>>
>> The recent changes in scipy.cluster have introduced some MemoryErrors
>>
>> ======================================================================
>> ERROR: Testing that kmeans2 init methods work.
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 137, in check_kmeans2_init
>>     kmeans2(data, 3, minit = 'random')
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 545, in kmeans2
>>     return _kmeans2(data, clusters, iter, nc)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 558, in _kmeans2
>>     label = vq(data, code)[0]
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 144, in vq
>>     results = _vq.vq(c_obs, c_code_book)
>> MemoryError
>>
>> ======================================================================
>> ERROR: Testing simple call to kmeans2 with rank 1 data.
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 129, in check_kmeans2_rank1
>>     code1 = kmeans2(data1, code, iter = 1)[0]
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 545, in kmeans2
>>     return _kmeans2(data, clusters, iter, nc)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 558, in _kmeans2
>>     label = vq(data, code)[0]
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 144, in vq
>>     results = _vq.vq(c_obs, c_code_book)
>> MemoryError
>>
>> ======================================================================
>> ERROR: Testing simple call to kmeans2 and its results.
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 114, in check_kmeans2_simple
>>     code1 = kmeans2(X, code, iter = 1)[0]
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 545, in kmeans2
>>     return _kmeans2(data, clusters, iter, nc)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 558, in _kmeans2
>>     label = vq(data, code)[0]
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 144, in vq
>>     results = _vq.vq(c_obs, c_code_book)
>> MemoryError
>>
>> ======================================================================
>> ERROR: This will cause kmean to have a cluster with no points.
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 108, in check_kmeans_lost_cluster
>>     res = kmeans(data, initk)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 400, in kmeans
>>     result = _kmeans(obs, guess, thresh = thresh)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 320, in _kmeans
>>     obs_code, distort = vq(obs, code_book)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 144, in vq
>>     results = _vq.vq(c_obs, c_code_book)
>> MemoryError
>>
>> ======================================================================
>> ERROR: check_kmeans_simple (scipy.cluster.tests.test_vq.test_kmean)
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 96, in check_kmeans_simple
>>     code1 = kmeans(X, code, iter = 1)[0]
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 400, in kmeans
>>     result = _kmeans(obs, guess, thresh = thresh)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 320, in _kmeans
>>     obs_code, distort = vq(obs, code_book)
>>   File "/usr/lib64/python2.4/site-packages/scipy/cluster/vq.py", line
>> 144, in vq
>>     results = _vq.vq(c_obs, c_code_book)
>> MemoryError
>>
>> ======================================================================
>> ERROR: check_vq (scipy.cluster.tests.test_vq.test_vq)
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 63, in check_vq
>>     label1, dist = _vq.vq(X, initc)
>> MemoryError
>>
>> ======================================================================
>> ERROR: Test special rank 1 vq algo, python implementation.
>> ----------------------------------------------------------------------
>> Traceback (most recent call last):
>>   File
>> "/usr/lib64/python2.4/site-packages/scipy/cluster/tests/test_vq.py",
>> line 85, in check_vq_1d
>>     a, b = _vq.vq(data, initc)
>> MemoryError
>>     
> That's regression is my fault :) I see that your machine is a 64 bits, 
> which may have exposed some bugs I didn't see. Unfortunately, I don't 
> have a 64 bits machine available right now...
>
> There is one obvious error I could spot, though: could you try this 
> simple patch ?
>
> Index: Lib/cluster/src/vq_module.c
> ===================================================================
> --- Lib/cluster/src/vq_module.c (revision 3110)
> +++ Lib/cluster/src/vq_module.c (working copy)
> @@ -1,5 +1,5 @@
>  /*
> - * Last Change: Tue Jun 19 11:00 PM 2007 J
> + * Last Change: Wed Jun 20 04:00 PM 2007 J
>   *
>   */
>  #include <Python.h>
> @@ -97,24 +97,24 @@
>              if (dist_a == NULL) {
>                  goto clean_code_a;
>              }
> -            index_a = (PyArrayObject*)PyArray_EMPTY(1, &n, NPY_INT, 0);
> +            index_a = (PyArrayObject*)PyArray_EMPTY(1, &n, 
> PyArray_INTP, 0);
>              if (index_a == NULL) {
>                  goto clean_dist_a;
>              }
>              float_tvq((float*)obs_a->data, (float*)code_a->data, n, nc, d,
> -                    (int*)index_a->data, (float*)dist_a->data);
> +                    (npy_intp*)index_a->data, (float*)dist_a->data);
>              break;
>          case NPY_DOUBLE:
>              dist_a = (PyArrayObject*)PyArray_EMPTY(1, &n, typenum1, 0);
>              if (dist_a == NULL) {
>                  goto clean_code_a;
>              }
> -            index_a = (PyArrayObject*)PyArray_EMPTY(1, &n, NPY_INT, 0);
> +            index_a = (PyArrayObject*)PyArray_EMPTY(1, &n, 
> PyArray_INTP, 0);
>              if (index_a == NULL) {
>                  goto clean_dist_a;
>              }
>              double_tvq((double*)obs_a->data, (double*)code_a->data, n, 
> nc, d,
> -                    (int*)index_a->data, (double*)dist_a->data);
> +                    (npy_intp*)index_a->data, (double*)dist_a->data);
>              break;
>          default:
>              PyErr_Format(PyExc_ValueError,
> @@ -151,4 +151,3 @@
>      Py_DECREF(obs_a);
>      return NULL;
>  }
> -
>
> David
> _______________________________________________
> Scipy-dev mailing list
> Scipy-dev at scipy.org
> http://projects.scipy.org/mailman/listinfo/scipy-dev
>   
 
Hi David,

I have applied your patch and installed scipy from scratch.
The errors persist.

Nils