numpy question (fairly basic, I think)

[Albert-Jan Roskam]

Hi,

I am new to numpy. I am reading binary data one record at a time (I have to) and I would like to store all the records in a numpy array which I pre-allocate. Below I try to fill the empty array with exactly one record, but it is filled with as many rows as there are columns. Why is this? It is probably something simple, but I am stuck! It is like the original record is not unpacked *as in tuple unpacking) into the array, so it remains one chunk, not an (nrows, ncols) structure.


from __future__ import print_function
import numpy as np


# one binary record
s = '\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00;@\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x80U\xe1@\x00\x00\x00\x00\x80\xd9\xe4@\x00\x00\x00\x00@\xa7\xe3@\xab\xaa\xaa\xaajG\xe3@\x00\x00\x00\x00\x80\xd9\xe4@\x00\x00\x00\x00\x00\x00;@\x00\x00\x00\x00\x00\x00;@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\xa4DI\tBx       qwertyuiopasdfghjklzxcvbnm,./                                                                           \x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00p\x9f@\x00\x00\x00\x00\x00\x00\x10@\x00\x00\x00\x00\x00\x00(@DEC 2012'


# read it into a structured array
formats = ['<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', '<d', 'a8', 'a104', '<d', '<d', '<d', '<d', 'a8']
names = ["v%02d" % i for i in range(len(formats))]
dt = np.dtype({'formats': formats, "names": names})
record = np.fromstring(s, dtype=dt)


# make it more compact
trunc_formats = ['f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'a1', 'a100', 'f4', 'f4', 'f4', 'f4', 'a8']
trunc_dt = np.dtype({'formats': trunc_formats, "names": names})
record = record.astype(trunc_dt)
print(record.shape)  # (1,), but it needs to be (1, 22)??
#record = np.asarray(*tuple(record.astype(trunc_dt)), dtype=np.object)
#record = np.expand_dims(record, axis=0)


# initialize an empty array and fill it with one record

nrows = 50  # arbitrary number
ncols = len(formats) # 22

array = np.zeros((nrows, ncols), trunc_dt)
array[0,:] = record
print(array)

# output: why is the record repeated ncols times?
[[ (1.0, 27.0, 1.0, 35500.0, 42700.0, 40250.0, 39483.33203125, 42700.0, 27.0, 27.0, 0.0, 1.0, 1.0, 1.0, 13575427072.0, 'x', 'qwertyuiopasdfghjklzxcvbnm,./                                                                       ', 1.0, 2012.0, 4.0, 12.0, 'DEC 2012')
(1.0, 27.0, 1.0, 35500.0, 42700.0, 40250.0, 39483.33203125, 42700.0, 27.0, 27.0, 0.0, 1.0, 1.0, 1.0, 13575427072.0, 'x', 'qwertyuiopasdfghjklzxcvbnm,./                                                                       ', 1.0, 2012.0, 4.0, 12.0, 'DEC 2012')
(1.0, 27.0, 1.0, 35500.0, 42700.0, 40250.0, 39483.33203125, 42700.0, 27.0, 27.0, 0.0, 1.0, 1.0, 1.0, 13575427072.0, 'x', 'qwertyuiopasdfghjklzxcvbnm,./                                                                       ', 1.0, 2012.0, 4.0, 12.0, 'DEC 2012')
...,
(1.0, 27.0, 1.0, 35500.0, 42700.0, 40250.0, 39483.33203125, 42700.0, 27.0, 27.0, 0.0, 1.0, 1.0, 1.0, 13575427072.0, 'x', 'qwertyuiopasdfghjklzxcvbnm,./                                                                       ', 1.0, 2012.0, 4.0, 12.0, 'DEC 2012')
(1.0, 27.0, 1.0, 35500.0, 42700.0, 40250.0, 39483.33203125, 42700.0, 27.0, 27.0, 0.0, 1.0, 1.0, 1.0, 13575427072.0, 'x', 'qwertyuiopasdfghjklzxcvbnm,./                                                                       ', 1.0, 2012.0, 4.0, 12.0, 'DEC 2012')
(1.0, 27.0, 1.0, 35500.0, 42700.0, 40250.0, 39483.33203125, 42700.0, 27.0, 27.0, 0.0, 1.0, 1.0, 1.0, 13575427072.0, 'x', 'qwertyuiopasdfghjklzxcvbnm,./                                                                       ', 1.0, 2012.0, 4.0, 12.0, 'DEC 2012')]
[ (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, '', '', 0.0, 0.0, 0.0, 0.0, '')
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, '', '', 0.0, 0.0, 0.0, 0.0, '')
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, '', '', 0.0, 0.0, 0.0, 0.0, '')
...,
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, '', '', 0.0, 0.0, 0.0, 0.0, '')
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, '', '', 0.0, 0.0, 0.0, 0.0, '')
(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, '', '', 0.0, 0.0, 0.0, 0.0, '')]
etc
etc

 
Thank you in advance!


Regards,

Albert-Jan




~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

All right, but apart from the sanitation, the medicine, education, wine, public order, irrigation, roads, a 

fresh water system, and public health, what have the Romans ever done for us?

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~