Unable to make http request to django server using twisted.web.client.AGENT

[Akshat Tripathi]

On Thursday, 5 November 2015 13:06:13 UTC+5:30, Akshat Tripathi  wrote:
> I have written a basic tcp server factory, server client and a service using twisted. The tcp server acts as the middleware between a django server and an another program (let's call it client program).
> 
> What I want to achieve -
> 
> 1.client requests the middleware tcp server;
> 
> 2.it sends a string as the request body;
> 
> 3.the middleware tcp server deserializes information from the request.
> 
> 4.middleware furthers the serialized information to the django server.
> 
> 5.The django server then responds to the middleware server which furthers the response to the client after serializing the response.
> 
> I am able to reach till step 3, but am unable to make any http request to the django server.
> 
> 
> 
> 
> The following is my middleware.py
> 
> from twisted.internet.protocol import ServerFactory
> from twisted.internet.protocol import Protocol
> 
> from test_service import MyService
> 
> 
> class TCPServerProtocol(Protocol):
>     data = ''
> 
>     def connectionMade(self):
>         self.deferred = self.factory.service.deferred
>         self.deferred.addCallback(self.factory.service.s)
>         self.deferred.addCallback(self.transport.write)
>         self.deferred.addBoth(lambda r: self.transport.loseConnection)
> 
>     def dataReceived(self, data):
>         self.data += data
> 
>     def connectionLost(self, reason):
>         self.forward(self.data)
> 
>     def forward(self, data):
>         if self.deferred is not None:
>             d, self.deferred = self.deferred, None
>             d.callback(data)
> 
> class TCPServerFactory(ServerFactory):
> 
>     protocol = TCPServerProtocol
> 
>     def __init__(self, service):
>         self.service = service
> 
> def runserver(ip, port):
>     iface =  {'home': '192.168.0.104', 'work': '127.0.0.1'}
>     service = MyService()
>     factory = TCPServerFactory(service)
>     from twisted.internet import reactor
>     reactor.listenTCP(port=port, factory=factory, interface=iface[ip])
>     reactor.run()
> 
> if __name__ == '__main__':
>     import sys
>     ip = sys.argv[1]
>     port = int(sys.argv[2])
>     runserver(ip, port)
> 
> 
> 
> 
> The following is test_service.py
> 
> from twisted.internet.defer import Deferred
> 
> from test_http_client import HTTPClientFactory
> 
> class MyService(object):
>     def __init__(self):
>         self.deferred = Deferred()
> 
>     def s(self, data):
>         kwargs = {}
>         kwargs['url'] = b'http://127.0.0.1:8000/some/end/point'
>         kwargs['url'] = kwargs['url'].encode('latin-1')
>         kwargs['method'] = 'POST'
>         kwargs['data'] = data
> 
>         client = HTTPClientFactory(**kwargs)
>         d = client.deferred
>         return d
> 
> 
> The following is test_http_client.py
> 
> from StringIO import StringIO
> import json
> 
> from twisted.internet.protocol import Protocol
> from twisted.internet.defer import Deferred
> from twisted.web.client import Agent, FileBodyProducer
> from twisted.web.http_headers import Headers
> 
> 
> class HTTPClientProtocol(Protocol):
>     def __init__(self, finished):
>         self.finished = finished
>         self.data = ''
> 
>     def dataReceived(self, data):
>         print '----------Data Received by HTTPClientProtocol----------'
>         print data
>         self.data += data
> 
>     def connectionLost(self, reason):
>         print '----------HTTP Client connection Lost----------'
>         print reason.getErrorMessage()
>         if self.finished is not None:
>             print 'finished is not None'
>             f, self.finished = self.finished, None
>             f.callback(self.data)
> 
> class HTTPClientFactory(object):
> 
>     """
>     Class handling communication with HTTP server.
>     """
> 
>     def __init__(self, **kwargs):
>         data = kwargs['data'] 
>         try:
>             body = FileBodyProducer(StringIO(json.dumps(data)))
>             print '----------Request body object created----------'
>         except Exception as e:
>             print '----------Request body object creation FAILURE----------'
>             print e
>             return e  
>         url = kwargs.get('url', None)
>         method = kwargs.get('method', None)
> 
>         from twisted.internet import reactor
>         agent = Agent(reactor)
>         if not data:
>             body = None
> 
>         self.deferred = agent.request(method,
>                                     url,
>                                     Headers({'Content-Type': ['application/json']}),
>                                     bodyProducer=body)
> 
>         self.deferred.addCallback(self.get_response)
> 
>     def get_response(self, response):
>         print 'Response received'
>         finished = Deferred()
>         response.deliverBody(HTTPClientProtocol(finished))
>         return finished

Found the bug. Below is the corrected code.

class TCPServerProtocol(Protocol):

    # data = ''

    def connectionMade(self):
        self.deferred = self.factory.service.deferred
        self.deferred.addCallback(self.factory.service.s)
        self.deferred.addCallback(self.transport.write)
        self.deferred.addBoth(lambda r: self.transport.loseConnection)

    def dataReceived(self, data):
        self.data = data           # instead of self.data += data
        self.forward(self.data)    # this is the right place to call this method which in turn fires the callback through which the request is made to the server.

    def connectionLost(self, reason): pass
        # this will not be called until the connection from client is ended. 
        # self.forward(self.data)

    def forward(self, data):
        if self.deferred is not None:
            d, self.deferred = self.deferred, None
            d.callback(data)