In Django 1.8, I have this view for users to post in and notify the user's followers of the post using a celery function, but it generates a rather confusing error:
dictionary changed size during iteration
Here is the view:
@login_required
def topic_reply(request, topic_id):
tform = PostForm()
topic = Topic.objects.get(pk=topic_id)
args = {}
posts = Post.objects.filter(topic= topic)
posts = Paginator(posts, DJANGO_SIMPLE_FORUM_REPLIES_PER_PAGE)
if request.method == 'POST':
post = PostForm(request.POST)
if post.is_valid():
p = post.save(commit = False)
p.topic = topic
p.title = post.cleaned_data['title']
p.body = post.cleaned_data['body']
p.creator = request.user
p.user_ip = request.META['REMOTE_ADDR']
if len(p.title)< 1:
p.title=p.body[:60]
p.save()
#notify followers of the new post creation
title = 'title' #topic.title
link = 'bla' #topic.slug
flwd = request.user
flwr_ids = FollowUser.objects.filter(followed=flwd).values('follower_id')
flwrs = User.objects.filter(id__in= flwr_ids).values('username','email')
notify_new_post.delay(flwd, flwrs , title, link) #<- here the is the problem
return HttpResponseRedirect('/forum/topic/%s/?page=%s' % (topic.slug, posts.num_pages))
else:
return HttpResponseRedirect('/forum/topic/%s/?page=%s' % (topic.slug, posts.num_pages))
else:
args.update(csrf(request))
args['form'] = tform
args['topic'] = topic
return render_to_response('myforum/reply.html', args,
context_instance=RequestContext(request))
This happens even before anything being passed to the function (I see nothing happen at celery daemon)
Here is the celery function:
#@app.task
@task()
def notify_new_post(flwd, flwrs, topic, link):
print 'post notification \n'
subject = 'New post'
from_email = '[email protected]'
#to_list = [email]
for f in flwrs:
to_email = f['email'].encode('ascii')
print "[to_email]: " , [to_email]
args = Context({
'flwd': flwd,
'recepient': f['username'],
'link': link
})
# if to_email !=[]:
plaintext = get_template('myforum/email_new_post.txt')
htmltext = get_template('myforum/email_new_post.html')
text_content = plaintext.render(args)
html_content = htmltext.render(args)
msg = EmailMultiAlternatives(subject, text_content, from_email, [to_email])
msg.attach_alternative(html_content, "text/html")
try:
msg.send()
print "[to_email]: " , [to_email]
print 'message sent! \n'
except Exception as e:
print '%s (%s)' % (e.message, type(e))
This is very odd to me because a very similar task for a 'topic' view works perfectly well. I've got really perplexed about this so appreciate your hints.
Update: here is the traceback
Environment:
Request Method: POST
Request URL: http://127.0.0.1:8000/forum/reply/52/
Django Version: 1.8.3
Python Version: 2.7.3
Installed Applications:
('django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.humanize',
'registration',
'aricle',
'photo',
'contact',
'captcha',
'pure_pagination',
'emoticons',
'debug_toolbar',
'django_markdown',
'myforum',
'userprofile',
'userpics',
'djcelery')
Installed Middleware:
(u'debug_toolbar.middleware.DebugToolbarMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'userprofile.middleware.ActiveUserMiddleware')
Traceback:
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/django/core/handlers/base.py" in get_response
132. response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/django/contrib/auth/decorators.py" in _wrapped_view
22. return view_func(request, *args, **kwargs)
File "/home/mypc/myproj/myforum/views.py" in topic_reply
315. notify_new_post.delay(flwd, flwrs , title, link)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/celery/app/task.py" in delay
453. return self.apply_async(args, kwargs)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/celery/app/task.py" in apply_async
555. **dict(self._get_exec_options(), **options)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/celery/app/base.py" in send_task
353. reply_to=reply_to or self.oid, **options
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/celery/app/amqp.py" in publish_task
305. **kwargs
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/messaging.py" in publish
161. compression, headers)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/messaging.py" in _prepare
237. body) = dumps(body, serializer=serializer)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/serialization.py" in dumps
164. payload = encoder(data)
File "/usr/lib/python2.7/contextlib.py" in __exit__
35. self.gen.throw(type, value, traceback)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/serialization.py" in _reraise_errors
59. reraise(wrapper, wrapper(exc), sys.exc_info()[2])
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/serialization.py" in _reraise_errors
55. yield
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/serialization.py" in dumps
164. payload = encoder(data)
File "/home/mypc/.projenv/local/lib/python2.7/site-packages/kombu/serialization.py" in pickle_dumps
356. return dumper(obj, protocol=pickle_protocol)
Exception Type: EncodeError at /forum/reply/52/
Exception Value: dictionary changed size during iteration
This is probably caused by the fact that the value you pass for flwrs is a django.db.models.query.ValuesQuerySet. Django query sets are evaluated lazily, which I'd expect is not good for serialization. Remember that everything you send to a Celery task and return from it has to be serialized. So it is inadvisable to pass anything else than simple types or types that you know for a fact are okay with being serialized (e.g. a class you designed yourself or one which you've checked inside and out to make sure will serialize cleanly).
The minimum fix I would suggest is passing list(flwrs) instead of flwrs. This would turn the query set into a plain list. I would also strongly suggest passing request.user.id as flwd instead of the user object itself. Passing ORM objects is a sure way to get unexpected behavior. (The Celery documentation mentions this.) Passing an id and reacquiring the object in the Celery task is the way to go.
However, when I look at the code overall, I don't see why the database accesses are performed in the view rather than in the Celery task. So unless there's a line or variable use I've missed somewhere, I'd change your code to pass only request.user.id as flwd and then perform the database accesses in the Celery task. So the view would invoke the task like this:
#notify followers of the new post creation
title = 'title' #topic.title
link = 'bla' #topic.slug
notify_new_post.delay(request.user.id, title, link)
And the task would start like this:
from django.contrib.auth import get_user_model
@task()
def notify_new_post(flwd_id, topic, link):
user_model = get_user_model()
flwd = user_model.objects.get(id=flwd_id)
flwr_ids = FollowUser.objects.filter(followed=flwd).values('follower_id')
flwrs = user_model.objects.filter(id__in= flwr_ids).values('username','email')
(Note regarding the last line: I've assumed that User is the user model that is used by your Django project so I'm using the return value of get_user_model() (assigned to user_model) rather than use User directly. If my assumption is incorrect and User is something else, then you'd have to use User as you originally did.)