ValueError: arrays must all be same length

[Tim Williams]

On Fri, Oct 2, 2020 at 11:00 AM Shaozhong SHI <shishaozhong at gmail.com>
wrote:

> Hello,
>
> I got a json response from an API and tried to use pandas to put data into
> a dataframe.
>
> However, I kept getting this ValueError: arrays must all be same length.
>
> Can anyone help?
>
> The following is the json text.  Regards, Shao
>
> (snip json_text)


> import pandas as pd
>
> import json
>
> j = json.JSONDecoder().decode(req.text)  ###req.json
>
> df = pd.DataFrame.from_dict(j)
>

I copied json_text into a Jupyter notebook and got the same error trying to
convert this into a pandas DataFrame:When I tried to copy this into a
string, I got an error,, but without enclosing the paste in quotes, I got
the dictionary.

dir(json_text)
['__class__',
 '__contains__',
 '__delattr__',
 '__delitem__',
 '__dir__',
 '__doc__',
 '__eq__',
 '__format__',
 '__ge__',
 '__getattribute__',
 '__getitem__',
 '__gt__',
 '__hash__',
 '__init__',
 '__init_subclass__',
 '__iter__',
 '__le__',
 '__len__',
 '__lt__',
 '__ne__',
 '__new__',
 '__reduce__',
 '__reduce_ex__',
 '__repr__',
 '__reversed__',
 '__setattr__',
 '__setitem__',
 '__sizeof__',
 '__str__',
 '__subclasshook__',
 'clear',
 'copy',
 'fromkeys',
 'get',
 'items',
 'keys',
 'pop',
 'popitem',
 'setdefault',
 'update',
 'values']

pd.DataFrame(json_text)

---------------------------------------------------------------------------

ValueError                                Traceback (most recent call last)
<ipython-input-21-88c1a0749403> in <module>
----> 1 pd.DataFrame(json_text)

D:\anaconda3\lib\site-packages\pandas\core\frame.py in __init__(self, data,
index, columns, dtype, copy)
    433             )
    434         elif isinstance(data, dict):
--> 435             mgr = init_dict(data, index, columns, dtype=dtype)
    436         elif isinstance(data, ma.MaskedArray):
    437             import numpy.ma.mrecords as mrecords

D:\anaconda3\lib\site-packages\pandas\core\internals\construction.py in
init_dict(data, index, columns, dtype)
    252             arr if not is_datetime64tz_dtype(arr) else arr.copy()
for arr in arrays
    253         ]
--> 254     return arrays_to_mgr(arrays, data_names, index, columns,
dtype=dtype)
    255
    256

D:\anaconda3\lib\site-packages\pandas\core\internals\construction.py in
arrays_to_mgr(arrays, arr_names, index, columns, dtype)
     62     # figure out the index, if necessary
     63     if index is None:
---> 64         index = extract_index(arrays)
     65     else:
     66         index = ensure_index(index)

D:\anaconda3\lib\site-packages\pandas\core\internals\construction.py in
extract_index(data)
    363             lengths = list(set(raw_lengths))
    364             if len(lengths) > 1:
--> 365                 raise ValueError("arrays must all be same length")
    366
    367             if have_dicts:

ValueError: arrays must all be same length


I got a different error trying json.loads(str(json_text)),
---------------------------------------------------------------------------
JSONDecodeError                           Traceback (most recent call last)
<ipython-input-29-8808d2d1b8b1> in <module>
----> 1 json.loads(str(json_text))

D:\anaconda3\lib\json\__init__.py in loads(s, cls, object_hook,
parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    355             parse_int is None and parse_float is None and
    356             parse_constant is None and object_pairs_hook is None
and not kw):
--> 357         return _default_decoder.decode(s)
    358     if cls is None:
    359         cls = JSONDecoder

D:\anaconda3\lib\json\decoder.py in decode(self, s, _w)
    335
    336         """
--> 337         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
    338         end = _w(s, end).end()
    339         if end != len(s):

D:\anaconda3\lib\json\decoder.py in raw_decode(self, s, idx)
    351         """
    352         try:
--> 353             obj, end = self.scan_once(s, idx)
    354         except StopIteration as err:
    355             raise JSONDecodeError("Expecting value", s, err.value)
from None

JSONDecodeError: Expecting property name enclosed in double quotes: line 1
column 2 (char 1)

I think the solution is to fix the arrays so that the lengths match.

for k in json_text.keys():
    if isinstance(json_text[k], list):
        print(k, len(json_text[k]))

relationships 0
locationTypes 0
regulatedActivities 2
gacServiceTypes 1
inspectionCategories 1
specialisms 4
inspectionAreas 0
historicRatings 4
reports 5

HTH,.

-- 
> https://mail.python.org/mailman/listinfo/python-list
>