This is the first long-polling application that I have ever built, and second project with Twisted, so I would appreciate any feedback that anyone has about anything in my code at all, as I may be going about this entirely the wrong way.
I have been cobbling together various examples as I've gone along and it's almost working, but I can't seem to find a way to get the data back to the Javascript. I have a Django site running on Twisted, and it seems to work fine so I'm not going to include the Django bits unless someone thinks it's important, and the only thing the Django site does is host a chat. I had originally set it up using regular polling, but I've been asked to change it to long-polling and I'm almost there (I hope).
Here's the HTML/JS (long.html):
<div class="chat-messages" style="width:300px;height:400px;border:1px solid black;overflow:scroll;" id="messages">
</div><br/>
<form action="javascript:sendMessage();" >
<input type="text" id="chat_nickname" name="author"/>
<input type="text" id="chat_input" name="message" class="chat-new"/>
<button class="submit">Submit</button>
</form>
</body>
<script type="text/javascript">
// keep track of the last time data wes received
var last_update = 0;
// call getData when the document has loaded
$(document).ready(function(){
getData(last_update);
});
// execute ajax call to chat_server.py
var getData = function(last_update){
$.ajax({
type: "GET",
url: "http://"+ window.location.hostname + ":8081?last_update=" + last_update + "&callback=?",
dataType: 'json',
async: true,
cache:false,
timeout: 300000,
success: function(response){
// append the new message to the message list
var messages = response.data.messages;
console.log(response);
for (i in messages){
$('<p><span class="time">[' + messages[i].time +']</span> - <span class="message">' + messages[i].message + '</span></p>').appendTo('#messages');
if (messages[i].time > last_update){
last_update = messages[i].time;
}
}
console.log("Last_update: " + last_update);
// Keep div scrolled to bottom
$("#messages").scrollTop($("#messages")[0].scrollHeight);
// Check again in a second
setTimeout('getData(' + last_update + ');', 1000);
},
error: function(XMLHttpRequest, textStatus, errorThrown){
// Try again in 10 seconds
setTimeout( "getData(" + last_update + ");", 10000);
},
failure: function(){ console.log('fail'); },
});
}
// Add a contribution to the conversation
function sendMessage(){
var nickname = $('#chat_nickname').val();
var message = $('#chat_input').val();
$('#chat_input').val("");
console.log( "nickname: " + nickname + "; message: " + message );
$.ajax({
type: 'POST',
url: '/chat/post_message/',
data: {
nickname: nickname,
message:message
},
success: function(data, status, xml){
console.log("Success! - " + status);
},
error: function(xml, status, error){
console.log(error + " - Error! - " + status);
},
complete: function(xml, status){
console.log("Complete! - " + status);
}
});
}
</script>
sendMessage passes data from the form to Django, and Django puts it into the database (and adds a time to it).
getData directs to :8081, where Twisted is listening with the ### Chat Server portion (second half) of this next bit of code (chat_server.py):
import datetime, json, sys, time, os, types
from twisted.web import client, resource, server, wsgi
from twisted.python import threadpool
from twisted.internet import defer, task, reactor
from twisted.application import internet, service
from twisted.enterprise import adbapi
from django.core.handlers.wsgi import WSGIHandler
## Django environment variables
sys.path.append("mydjangosite")
os.environ['DJANGO_SETTINGS_MODULE'] = 'mydjangosite.settings'
## Tying Django's WSGIHandler into Twisted
def wsgi_resource():
pool = threadpool.ThreadPool()
pool.start()
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
wsgi_resource = wsgi.WSGIResource(reactor, pool, WSGIHandler())
return wsgi_resource
## Twisted Application Framework
application = service.Application('twisted-django')
class Root(resource.Resource):
def __init__(self, wsgi_resource = None):
resource.Resource.__init__(self)
if wsgi_resource != None:
self.wsgi_resource = wsgi_resource
def getChild(self, path, request):
child_path = request.prepath.pop(0)
request.postpath.insert(0, child_path)
return self.wsgi_resource
def render_GET(self, request):
id = request.args.get('id', [""])[0]
command = request.args.get('command', [""])[0]
self.get_page(request, id)
return server.NOT_DONE_YET
@defer.inlineCallbacks
def get_page(self, request, id):
page = yield client.getPage("/chat/latest/%s" % id)
request.write(page)
request.finish()
## Create and attach the django site to the reactor
django_root = Root(wsgi_resource())
django_factory = server.Site(django_root)
reactor.listenTCP(8080, django_factory)
### Chat Server
class ChatServer(resource.Resource):
isLeaf = True
def __init__(self):
# throttle in seconds
self.throttle = 5
# store client requests
self.delayed_requests = []
# setup a loop to process collected requests
loopingCall = task.LoopingCall(self.processDelayedRequests)
loopingCall.start(self.throttle, False)
# Initialize
resource.Resource.__init__(self)
def render(self, request):
"""Handle a new request"""
request.setHeader('Content-Type', 'applicaton/json')
args = request.args
# set jsonp callback handler name if it exists
if 'callback' in args:
request.jsonpcallback = args['callback'][0]
# set last_update if it exists
if 'last_update' in args:
request.last_update = args ['last_update'][0]
data = self.getData(request)
if type(data) is not types.InstanceType and len(data) > 0:
# send the requested messages back
return self.__format_response(request, 1, data)
else:
# or put them in the delayed request list and keep the connection going
self.delayed_requests.append(request)
return server.NOT_DONE_YET
def getData(self, request):
data = {}
dbpool = adbapi.ConnectionPool("sqlite3", database="/home/server/development/twisted_chat/twisted-wsgi-django/mydjangosite/site.db", check_same_thread=False)
last_update = request.last_update
print "LAST UPDATE: ", last_update
new_messages = dbpool.runQuery("SELECT * FROM chat_message WHERE time > %r" % request.last_update )
return new_messages.addCallback(self.gotRows, request )
def gotRows(self, rows, request):
if rows:
data = {"messages":
[{ 'author': row[1], 'message':row[2],'timestamp': row[3] } for row in rows]
}
print 'MESSAGES: ', data
if len(data) > 0:
return self.__format_response(request, 1, data)
return data
def processDelayedRequests(self):
for request in self.delayed_requests:
data = self.getData(request)
if type(data) is not types.InstanceType and len(data) > 0:
try:
print "REQUEST DATA:", data
request.write(self.__format_response(request, 1, data))
request.finish()
except:
print 'connection lost before complete.'
finally:
self.delayed_requests.remove(request)
def __format_response(self, request, status, data):
response = json.dumps({ "status": status, "time": int(time.time()), "data": data })
if hasattr(request, 'jsonpcallback'):
return request.jsonpcallback + '(' + response + ')'
else:
return response
chat_server = ChatServer()
chat_factory = server.Site(chat_server)
reactor.listenTCP(8081, chat_factory)
Here, render tries to getData (it may not ever?) and when it can't puts the request into self.delayed_requests. getData uses enterprise.adbapi to make a query on Django's db, returning a Deferred instance. processedDelayedRequests goes through that the delayed request queue and, if the query is completed, that data gets passed to gotRows, which then converts it to the format I want and sends it off to __format_response which sends the data back to the JS where it can be dealt with. That's the theory anyway - the previous sentence is where I think my problem is.
print "LAST UPDATE: ", last_update always prints "LAST_UPDATE: 0" but last_update gets updated via the JS, so this not an error.
print 'MESSAGES: ', data prints "{'messages': [{'timestamp': u'2013-08-10 16:59:07.909350', 'message': u'chat message', 'author': u'test'}, {'timestamp': u'2013-08-10 17:11:56.893340', 'message': u'hello', 'author': u'pardon'}]}" and so forth as new messages are added to the db. It gets new data when posts are made, and seems to work pretty well otherwise.
print "REQUEST DATA:", data never fires at all... I think this method is leftover from an earlier attempt to get this working.
I get the correct output from gotRows but don't know how to get that output passed back to the client. I'm not even half confident with my understanding of Deferreds, so I think that's where my problem lies, but I don't know what I can do to move forward from here. Any help would be very much appreciated.
Sometimes, a function in a twisted application may conditionally return data,
and at other times return a Deferred. In those cases, you can't check to see
if you got data; you probably won't, and in the cases that you do get a
deferred, no amount of rechecking will change that; you must always turn such
functions into real Deferreds, with maybeDeferred, and then attach a
callback to the result.
That said, t.e.adbapi.ConnectionPool.runQuery() is not such a function.
it always returns a deferred. The only way to work with that data is to
attach a callback. In general, you won't ever see the result of an
asynchronous call in twisted applications in the same function that makes the
initial call.
This means that, since you want to run a query for every long polling request, and since those are unconditionally asynchronous (you have to return from your render() function before they can even start), your render() always returns NOT_DONE_YET:
def render(self, request):
"""Handle a new request"""
request.setHeader('Content-Type', 'applicaton/json')
self.getData(request)
return server.NOT_DONE_YET
and now everything needs to happen correctly in getData. As it turns out, the
handling of the deferred from runQuery is fine; but the sql itself has a
pretty big issue. To understand why, imagine a clever hacker tried to access
http://yoursite?last_update=5+and+"secret"+in+(select+password+from+users)
The fix is easy, though, don't do string interpolation, use bind parameters.
switch the %s for a ? in the query, and the % for a , in the function
call itself. While we're at it, lets move the ConnectionPool out of this method and into __init__, you don't want or need a whole pool for each retry for each request.
def getData(self, request):
last_update = request.args['last_update']
print "LAST UPDATE: ", last_update
new_messages = self.dbpool.runQuery("SELECT *"
" FROM chat_message"
" WHERE time > ?", request.last_update)
# ^ ^
return new_messages.addCallback(self.gotRows, request)
The callback attached to the deferred returned by runQuery is returning the
formatted result; but there's nobody to return it to; it needs to do all the
work itself. Fortunately, we're already giving it the request to work with,
so that's not too difficult. We also need to handle the case of when there was
no data to return, since there's nobody on the other end to add it to the list
of delayed requests.
def gotRows(self, rows, request):
if rows:
# we have data to send back to the client! actually finish the
# request here.
data = {"messages": [{'author': row[1], 'message': row[2], 'timestamp': row[3]} for
row in rows]}
request.write(self.__format_response(request, 1, data))
request.finish()
else:
self.delayed_requests.append(self)
Lastly, we need to make a similar change to processedDelayedRequests() as we
made in render(). it can only fire off the query, it can't update its state
based on the results because it doesn't have them. To simplify things, we'll
just eat items off the list.
def processDelayedRequests(self):
delayed_requests = self.delayed_requests
self.delayed_requests = []
while self.delayed_requests:
# grab a request out of the "queue"
request = self.delayed_requests.pop()
# we can cause another attempt at getting data, but we'll never get
# to see what hapened with it in this function.
self.getData(request)