how to send a json of yield list

meInvent bbird jobmattcon at gmail.com
Thu Oct 13 22:14:41 EDT 2016


after google a several solutions, 

First method i searched has memory error
sock.send(json.dumps(StreamArray()))
Traceback (most recent call last):
  File "pusher.py", line 43, in <module>
    sock.send(json.dumps(StreamArray()))
  File "C:\Python27\lib\json\__init__.py", line 243, in dumps
    return _default_encoder.encode(obj)
  File "C:\Python27\lib\json\encoder.py", line 207, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "C:\Python27\lib\json\encoder.py", line 270, in iterencode
    return _iterencode(o, 0)
MemoryError

if use this solution, got another error

combobject = getcombinations()
sock.send(json.dumps(combobject, cls=PythonObjectEncoder))

C:\Users\martlee2\Downloads>python pusher.py tcp://*:8080
Traceback (most recent call last):
  File "pusher.py", line 42, in <module>
    sock.send(json.dumps(combobject, cls=PythonObjectEncoder))
  File "C:\Python27\lib\json\__init__.py", line 250, in dumps
    sort_keys=sort_keys, **kw).encode(obj)
  File "C:\Python27\lib\json\encoder.py", line 207, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "C:\Python27\lib\json\encoder.py", line 270, in iterencode
    return _iterencode(o, 0)
  File "pusher.py", line 13, in default
    return {'_python_object': pickle.dumps(obj)}
  File "C:\Python27\lib\pickle.py", line 1374, in dumps
    Pickler(file, protocol).dump(obj)
  File "C:\Python27\lib\pickle.py", line 224, in dump
    self.save(obj)
  File "C:\Python27\lib\pickle.py", line 306, in save
    rv = reduce(self.proto)
  File "C:\Python27\lib\copy_reg.py", line 70, in _reduce_ex
    raise TypeError, "can't pickle %s objects" % base.__name__
TypeError: can't pickle generator objects



#python pusher.py tcp://*:8080
import sys
import time
import zmq
import json
from json import dumps, loads, JSONEncoder, JSONDecoder
import pickle

class PythonObjectEncoder(JSONEncoder):
    def default(self, obj):
        if isinstance(obj, (list, dict, str, unicode, int, float, bool, type(None))):
            return JSONEncoder.default(self, obj)
        return {'_python_object': pickle.dumps(obj)}

def as_python_object(dct):
    if '_python_object' in dct:
        return pickle.loads(str(dct['_python_object']))
    return dct

context = zmq.Context()
sock = context.socket(zmq.PUSH)
sock.bind(sys.argv[1])

def getcombinations():
    for ii in range(1,2000):
        for jj in range(1,2000):
            for kk in range(1,2000):
                yield [ii,jj,kk]

class StreamArray(list):
    def __iter__(self):
        return getcombinations()

    # according to the comment below
    def __len__(self):
        return 1

while True:
    time.sleep(1)
    #sock.send(sys.argv[1] + ':' + time.ctime())
    combobject = getcombinations()
    sock.send(json.dumps(combobject, cls=PythonObjectEncoder))



puller.py

#python puller.py tcp://localhost:8080
import sys
import zmq
import json

context = zmq.Context()
sock = context.socket(zmq.PULL)

for arg in sys.argv[1:]:
    sock.connect(arg)

while True:
    message = sock.recv()
    combinations = json.loads(message)
    print(str(combinations))



More information about the Python-list mailing list