[Numpy-svn] r4666 - in trunk/numpy/lib: . src tests
numpy-svn at scipy.org
numpy-svn at scipy.org
Sun Dec 30 04:32:36 EST 2007
Author: oliphant
Date: 2007-12-30 03:32:31 -0600 (Sun, 30 Dec 2007)
New Revision: 4666
Added:
trunk/numpy/lib/_datasource.py
trunk/numpy/lib/format.py
trunk/numpy/lib/io.py
trunk/numpy/lib/tests/test_format.py
Modified:
trunk/numpy/lib/__init__.py
trunk/numpy/lib/src/_compiled_base.c
trunk/numpy/lib/utils.py
Log:
Merge the lib_for_io branch back into the trunk.
Modified: trunk/numpy/lib/__init__.py
===================================================================
--- trunk/numpy/lib/__init__.py 2007-12-30 03:36:50 UTC (rev 4665)
+++ trunk/numpy/lib/__init__.py 2007-12-30 09:32:31 UTC (rev 4666)
@@ -15,6 +15,7 @@
#import convertcode
from utils import *
from arraysetops import *
+from io import *
import math
__all__ = ['emath','math']
@@ -29,6 +30,7 @@
__all__ += getlimits.__all__
__all__ += utils.__all__
__all__ += arraysetops.__all__
+__all__ += io.__all__
def test(level=1, verbosity=1):
from numpy.testing import NumpyTest
Copied: trunk/numpy/lib/_datasource.py (from rev 4664, branches/lib_for_io/_datasource.py)
Copied: trunk/numpy/lib/format.py (from rev 4664, branches/lib_for_io/format.py)
Copied: trunk/numpy/lib/io.py (from rev 4664, branches/lib_for_io/io.py)
Modified: trunk/numpy/lib/src/_compiled_base.c
===================================================================
--- trunk/numpy/lib/src/_compiled_base.c 2007-12-30 03:36:50 UTC (rev 4665)
+++ trunk/numpy/lib/src/_compiled_base.c 2007-12-30 09:32:31 UTC (rev 4666)
@@ -9,6 +9,9 @@
goto fail;} \
}
+#define PYSETERROR(message) \
+{ PyErr_SetString(ErrorObject, message); goto fail; }
+
static intp
incr_slot_ (double x, double *bins, intp lbins)
{
@@ -526,6 +529,269 @@
return Py_None;
}
+
+static char packbits_doc[] =
+ "out = numpy.packbits(myarray, axis=None)\n\n"
+ " myarray : an integer type array whose elements should be packed to bits\n\n"
+ " This routine packs the elements of a binary-valued dataset into a\n"
+ " NumPy array of type uint8 ('B') whose bits correspond to\n"
+ " the logical (0 or nonzero) value of the input elements.\n"
+ " The dimension over-which bit-packing is done is given by axis.\n"
+ " The shape of the output has the same number of dimensions as the input\n"
+ " (unless axis is None, in which case the output is 1-d).\n"
+ "\n"
+ " Example:\n"
+ " >>> a = array([[[1,0,1],\n"
+ " ... [0,1,0]],\n"
+ " ... [[1,1,0],\n"
+ " ... [0,0,1]]])\n"
+ " >>> b = numpy.packbits(a,axis=-1)\n"
+ " >>> b\n"
+ " array([[[160],[64]],[[192],[32]]], dtype=uint8)\n\n"
+ " Note that 160 = 128 + 32\n"
+ " 192 = 128 + 64\n";
+
+static char unpackbits_doc[] =
+ "out = numpy.unpackbits(myarray, axis=None)\n\n"
+ " myarray - array of uint8 type where each element represents a bit-field\n"
+ " that should be unpacked into a boolean output array\n\n"
+ " The shape of the output array is either 1-d (if axis is None) or\n"
+ " the same shape as the input array with unpacking done along the\n"
+ " axis specified.";
+
+/* PACKBITS
+
+ This function packs binary (0 or 1) 1-bit per pixel arrays
+ into contiguous bytes.
+
+*/
+
+static void
+_packbits(
+ void *In,
+ int element_size, /* in bytes */
+ npy_intp in_N,
+ npy_intp in_stride,
+ void *Out,
+ npy_intp out_N,
+ npy_intp out_stride
+ )
+{
+ char build;
+ int i, index;
+ npy_intp out_Nm1;
+ int maxi, remain, nonzero, j;
+ char *outptr,*inptr;
+
+ outptr = Out; /* pointer to output buffer */
+ inptr = In; /* pointer to input buffer */
+
+ /* Loop through the elements of In */
+ /* Determine whether or not it is nonzero.
+ Yes: set correspdoning bit (and adjust build value)
+ No: move on
+ /* Every 8th value, set the value of build and increment the outptr */
+
+ remain = in_N % 8; /* uneven bits */
+ if (remain == 0) remain = 8;
+ out_Nm1 = out_N - 1;
+ for (index = 0; index < out_N; index++) {
+ build = 0;
+ maxi = (index != out_Nm1 ? 8 : remain);
+ for (i = 0; i < maxi ; i++) {
+ build <<= 1; /* shift bits left one bit */
+ nonzero = 0;
+ for (j = 0; j < element_size; j++) /* determine if this number is non-zero */
+ nonzero += (*(inptr++) != 0);
+ inptr += (in_stride - element_size); /* advance to next input */
+ build += (nonzero != 0); /* add to this bit if the input value is non-zero */
+ }
+ if (index == out_Nm1) build <<= (8-remain);
+ /* printf("Here: %d %d %d %d\n",build,slice,index,maxi);
+ */
+ *outptr = build;
+ outptr += out_stride;
+ }
+ return;
+}
+
+
+static void
+_unpackbits(
+ void *In,
+ int el_size, /* unused */
+ npy_intp in_N,
+ npy_intp in_stride,
+ void *Out,
+ npy_intp out_N,
+ npy_intp out_stride
+ )
+{
+ unsigned char mask;
+ int i,index;
+ char *inptr, *outptr;
+
+ /* Loop through the elements of out
+ */
+ outptr = Out;
+ inptr = In;
+ for (index = 0; index < in_N; index++) {
+ mask = 128;
+ for (i = 0; i < 8 ; i++) {
+ *outptr = ((mask & (unsigned char)(*inptr)) != 0);
+ outptr += out_stride;
+ mask >>= 1;
+ }
+ inptr += in_stride;
+ }
+ return;
+}
+
+static PyObject *
+pack_or_unpack_bits(PyObject *input, int axis, int unpack)
+{
+ PyArrayObject *inp;
+ PyObject *new=NULL;
+ PyObject *out=NULL;
+ npy_intp outdims[MAX_DIMS];
+ int i;
+ void (*thefunc)(void *, int, npy_intp, npy_intp, void *, npy_intp, npy_intp);
+ PyArrayIterObject *it, *ot;
+
+ inp = (PyArrayObject *)PyArray_FROM_O(input);
+
+ if (inp == NULL) return NULL;
+
+ if (unpack) {
+ if (PyArray_TYPE(inp) != NPY_UBYTE)
+ PYSETERROR("Expecting an input array of unsigned byte data type");
+ }
+ else {
+ if (!PyArray_ISINTEGER(inp))
+ PYSETERROR("Expecting an input array of integer data type");
+ }
+
+ new = PyArray_CheckAxis(inp, &axis, 0);
+ Py_DECREF(inp);
+ if (new == NULL) return NULL;
+
+ /* Handle zero-dim array separately */
+ if (PyArray_SIZE(new) == 0) {
+ return PyArray_Copy((PyArrayObject *)new);
+ }
+
+ if (PyArray_NDIM(new) == 0) {
+ if (unpack) {
+ /* Handle 0-d array by converting it to a 1-d array */
+ PyObject *temp;
+ PyArray_Dims newdim = {NULL, 1};
+ npy_intp shape=1;
+ newdim.ptr = &shape;
+ temp = PyArray_Newshape((PyArrayObject *)new, &newdim, NPY_CORDER);
+ if (temp == NULL) goto fail;
+ Py_DECREF(new);
+ new = temp;
+ }
+ else {
+ ubyte *optr, *iptr;
+ out = PyArray_New(new->ob_type, 0, NULL, NPY_UBYTE,
+ NULL, NULL, 0, 0, NULL);
+ if (out == NULL) goto fail;
+ optr = PyArray_DATA(out);
+ iptr = PyArray_DATA(new);
+ *optr = 0;
+ for (i=0; i<PyArray_ITEMSIZE(new); i++) {
+ if (*iptr != 0) {
+ *optr = 1;
+ break;
+ }
+ iptr++;
+ }
+ goto finish;
+ }
+ }
+
+
+ /* Setup output shape */
+ for (i=0; i<PyArray_NDIM(new); i++) {
+ outdims[i] = PyArray_DIM(new, i);
+ }
+
+ if (unpack) {
+ /* Multiply axis dimension by 8 */
+ outdims[axis] <<= 3;
+ thefunc = _unpackbits;
+ }
+ else {
+ /* Divide axis dimension by 8 */
+ /* 8 -> 1, 9 -> 2, 16 -> 2, 17 -> 3 etc.. */
+ outdims[axis] = ((outdims[axis] - 1) >> 3) + 1;
+ thefunc = _packbits;
+ }
+
+ /* Create output array */
+ out = PyArray_New(new->ob_type, PyArray_NDIM(new), outdims, PyArray_UBYTE,
+ NULL, NULL, 0, PyArray_ISFORTRAN(new), NULL);
+ if (out == NULL) goto fail;
+
+ /* Setup iterators to iterate over all but given axis */
+ it = (PyArrayIterObject *)PyArray_IterAllButAxis((PyObject *)new, &axis);
+ ot = (PyArrayIterObject *)PyArray_IterAllButAxis((PyObject *)out, &axis);
+ if (it == NULL || ot == NULL) {
+ Py_XDECREF(it);
+ Py_XDECREF(ot);
+ goto fail;
+ }
+
+ while(PyArray_ITER_NOTDONE(it)) {
+ thefunc(PyArray_ITER_DATA(it), PyArray_ITEMSIZE(new),
+ PyArray_DIM(new, axis), PyArray_STRIDE(new, axis),
+ PyArray_ITER_DATA(ot), PyArray_DIM(out, axis),
+ PyArray_STRIDE(out, axis));
+ PyArray_ITER_NEXT(it);
+ PyArray_ITER_NEXT(ot);
+ }
+ Py_DECREF(it);
+ Py_DECREF(ot);
+
+ finish:
+ Py_DECREF(new);
+ return out;
+
+ fail:
+ Py_XDECREF(new);
+ Py_XDECREF(out);
+ return NULL;
+}
+
+
+
+static PyObject *
+io_pack(PyObject *self, PyObject *args, PyObject *kwds)
+{
+ PyObject *obj;
+ int axis=NPY_MAXDIMS;
+ static char *kwlist[] = {"in", "axis", NULL};
+
+ if (!PyArg_ParseTupleAndKeywords( args, kwds, "O|O&" , kwlist,
+ &obj, PyArray_AxisConverter, &axis))
+ return NULL;
+ return pack_or_unpack_bits(obj, axis, 0);
+}
+
+static PyObject *
+io_unpack(PyObject *self, PyObject *args, PyObject *kwds)
+{
+ PyObject *obj;
+ int axis=NPY_MAXDIMS;
+ static char *kwlist[] = {"in", "axis", NULL};
+
+ if (!PyArg_ParseTupleAndKeywords( args, kwds, "O|O&" , kwlist,
+ &obj, PyArray_AxisConverter, &axis))
+ return NULL;
+ return pack_or_unpack_bits(obj, axis, 1);
+}
+
static struct PyMethodDef methods[] = {
{"_insert", (PyCFunction)arr_insert, METH_VARARGS | METH_KEYWORDS,
arr_insert__doc__},
@@ -537,6 +803,10 @@
NULL},
{"add_docstring", (PyCFunction)arr_add_docstring, METH_VARARGS,
NULL},
+ {"packbits", (PyCFunction)io_pack, METH_VARARGS | METH_KEYWORDS,
+ packbits_doc},
+ {"unpackbits", (PyCFunction)io_unpack, METH_VARARGS | METH_KEYWORDS,
+ unpackbits_doc},
{NULL, NULL} /* sentinel */
};
@@ -578,7 +848,7 @@
PyDict_SetItemString(d, "__version__", s);
Py_DECREF(s);
- ErrorObject = PyString_FromString("numpy.lib._compiled_base.error");
+ ErrorObject = PyString_FromString("numpy.lib.error");
PyDict_SetItemString(d, "error", ErrorObject);
Py_DECREF(ErrorObject);
Copied: trunk/numpy/lib/tests/test_format.py (from rev 4664, branches/lib_for_io/tests/test_format.py)
Modified: trunk/numpy/lib/utils.py
===================================================================
--- trunk/numpy/lib/utils.py 2007-12-30 03:36:50 UTC (rev 4665)
+++ trunk/numpy/lib/utils.py 2007-12-30 09:32:31 UTC (rev 4666)
@@ -1,3 +1,4 @@
+import compiler
import os
import sys
import inspect
@@ -7,9 +8,10 @@
from numpy.core import product, ndarray
__all__ = ['issubclass_', 'get_numpy_include', 'issubsctype',
- 'issubdtype', 'deprecate', 'get_numarray_include',
+ 'issubdtype', 'deprecate', 'deprecate_with_doc',
+ 'get_numarray_include',
'get_include', 'info', 'source', 'who',
- 'byte_bounds', 'may_share_memory']
+ 'byte_bounds', 'may_share_memory', 'safe_eval']
def issubclass_(arg1, arg2):
try:
@@ -82,15 +84,32 @@
func.__name__ = name
return func
-def deprecate(func, oldname, newname):
+def deprecate(func, oldname=None, newname=None):
+ """Deprecate old functions.
+ Issues a DeprecationWarning, adds warning to oldname's docstring,
+ rebinds oldname.__name__ and returns new function object.
+
+ Example:
+ oldfunc = deprecate(newfunc, 'oldfunc', 'newfunc')
+
+ """
+
import warnings
+ if oldname is None:
+ oldname = func.func_name
+ if newname is None:
+ str1 = "%s is deprecated" % (oldname,)
+ depdoc = "%s is DEPRECATED!" % (oldname,)
+ else:
+ str1 = "%s is deprecated, use %s" % (oldname, newname),
+ depdoc = '%s is DEPRECATED! -- use %s instead' % (oldname, newname,)
+
def newfunc(*args,**kwds):
- warnings.warn("%s is deprecated, use %s" % (oldname, newname),
- DeprecationWarning)
+ warnings.warn(str1, DeprecationWarning)
return func(*args, **kwds)
+
newfunc = _set_function_name(newfunc, oldname)
doc = func.__doc__
- depdoc = '%s is DEPRECATED in numpy: use %s instead' % (oldname, newname,)
if doc is None:
doc = depdoc
else:
@@ -104,6 +123,24 @@
newfunc.__dict__.update(d)
return newfunc
+def deprecate_with_doc(somestr):
+ """Decorator to deprecate functions and provide detailed documentation
+ with 'somestr' that is added to the functions docstring.
+
+ Example:
+ depmsg = 'function numpy.lib.foo has been merged into numpy.lib.io.foobar'
+ @deprecate_with_doc(depmsg)
+ def foo():
+ pass
+
+ """
+
+ def _decorator(func):
+ newfunc = deprecate(func)
+ newfunc.__doc__ += "\n" + somestr
+ return newfunc
+ return _decorator
+
get_numpy_include = deprecate(get_include, 'get_numpy_include', 'get_include')
@@ -430,3 +467,113 @@
print >> output, inspect.getsource(object)
except:
print >> output, "Not available for this object."
+
+#-----------------------------------------------------------------------------
+
+# The following SafeEval class and company are adapted from Michael Spencer's
+# ASPN Python Cookbook recipe:
+# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/364469
+# Accordingly it is mostly Copyright 2006 by Michael Spencer.
+# The recipe, like most of the other ASPN Python Cookbook recipes was made
+# available under the Python license.
+# http://www.python.org/license
+
+# It has been modified to:
+# * handle unary -/+
+# * support True/False/None
+# * raise SyntaxError instead of a custom exception.
+
+class SafeEval(object):
+
+ def visit(self, node, **kw):
+ cls = node.__class__
+ meth = getattr(self,'visit'+cls.__name__,self.default)
+ return meth(node, **kw)
+
+ def default(self, node, **kw):
+ raise SyntaxError("Unsupported source construct: %s" % node.__class__)
+
+ def visitExpression(self, node, **kw):
+ for child in node.getChildNodes():
+ return self.visit(child, **kw)
+
+ def visitConst(self, node, **kw):
+ return node.value
+
+ def visitDict(self, node,**kw):
+ return dict([(self.visit(k),self.visit(v)) for k,v in node.items])
+
+ def visitTuple(self, node, **kw):
+ return tuple([self.visit(i) for i in node.nodes])
+
+ def visitList(self, node, **kw):
+ return [self.visit(i) for i in node.nodes]
+
+ def visitUnaryAdd(self, node, **kw):
+ return +self.visit(node.getChildNodes()[0])
+
+ def visitUnarySub(self, node, **kw):
+ return -self.visit(node.getChildNodes()[0])
+
+ def visitName(self, node, **kw):
+ if node.name == 'False':
+ return False
+ elif node.name == 'True':
+ return True
+ elif node.name == 'None':
+ return None
+ else:
+ raise SyntaxError("Unknown name: %s" % node.name)
+
+def safe_eval(source):
+ """ Evaluate a string containing a Python literal expression without
+ allowing the execution of arbitrary non-literal code.
+
+ Parameters
+ ----------
+ source : str
+
+ Returns
+ -------
+ obj : object
+
+ Raises
+ ------
+ SyntaxError if the code is invalid Python expression syntax or if it
+ contains non-literal code.
+
+ Examples
+ --------
+ >>> from numpy.lib.utils import safe_eval
+ >>> safe_eval('1')
+ 1
+ >>> safe_eval('[1, 2, 3]')
+ [1, 2, 3]
+ >>> safe_eval('{"foo": ("bar", 10.0)}')
+ {'foo': ('bar', 10.0)}
+ >>> safe_eval('import os')
+ Traceback (most recent call last):
+ ...
+ SyntaxError: invalid syntax
+ >>> safe_eval('open("/home/user/.ssh/id_dsa").read()')
+ Traceback (most recent call last):
+ ...
+ SyntaxError: Unsupported source construct: compiler.ast.CallFunc
+ >>> safe_eval('dict')
+ Traceback (most recent call last):
+ ...
+ SyntaxError: Unknown name: dict
+ """
+ walker = SafeEval()
+ try:
+ ast = compiler.parse(source, "eval")
+ except SyntaxError, err:
+ raise
+ try:
+ return walker.visit(ast)
+ except SyntaxError, err:
+ raise
+
+#-----------------------------------------------------------------------------
+
+
More information about the Numpy-svn
mailing list