I work with django 1.10.5 and celery 4.0.2. I have the structure like this.
-proj
-application
__init__.py
celery.py
celery_conf.py
settings.py
tasks.py
urls.py
wsgi.py
-extuser (application with extended UserModel)
-pages
-migrations
-templates
__init__.py
admin.py
apps.py
forms.py
models.py
tasks.py
tests.py
views.py
I have the task which uses model from file which contains my extended user model. If I try to run celery with this command:
celery -A application worker -l INFO -B
Then I see this error:
Traceback (most recent call last):
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/bin/celery", line 11, in <module>
sys.exit(main())
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/__main__.py", line 14, in main
_main()
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/celery.py", line 326, in main
cmd.execute_from_commandline(argv)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/celery.py", line 488, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/base.py", line 281, in execute_from_commandline
return self.handle_argv(self.prog_name, argv[1:])
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/celery.py", line 480, in handle_argv
return self.execute(command, argv)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/celery.py", line 412, in execute
).run_from_argv(self.prog_name, argv[1:], command=argv[0])
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/worker.py", line 221, in run_from_argv
return self(*args, **options)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/base.py", line 244, in __call__
ret = self.run(*args, **kwargs)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/bin/worker.py", line 255, in run
**kwargs)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/worker/worker.py", line 94, in __init__
self.app.loader.init_worker()
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/loaders/base.py", line 116, in init_worker
self.import_default_modules()
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/loaders/base.py", line 111, in import_default_modules
return [self.import_task_module(m) for m in self.default_modules]
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/loaders/base.py", line 97, in import_task_module
return self.import_from_cwd(module)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/loaders/base.py", line 106, in import_from_cwd
package=package,
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/celery/loaders/base.py", line 100, in import_module
return importlib.import_module(module, package=package)
File "/usr/lib64/python2.7/importlib/__init__.py", line 37, in import_module
__import__(name)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/src/pages/tasks.py", line 4, in <module>
from extuser.models import UserLinksArchive
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/src/extuser/models.py", line 5, in <module>
from django.contrib.auth.models import AbstractBaseUser
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/django/contrib/auth/models.py", line 4, in <module>
from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/django/contrib/auth/base_user.py", line 52, in <module>
class AbstractBaseUser(models.Model):
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/django/db/models/base.py", line 105, in __new__
app_config = apps.get_containing_app_config(module)
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/django/apps/registry.py", line 237, in get_containing_app_config
self.check_apps_ready()
File "/home/dmitriy/MyEnv/DjangoProjects/sPrint/env/local/lib/python2.7/site-packages/django/apps/registry.py", line 124, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
File pages/tasks.py looks like this:
from __future__ import absolute_import, unicode_literals
from application.celery import app
from extuser.models import UserLinksArchive
@app.task
def link_destroy_task(link_id):
link = UserLinksArchive.objects.get(id=link_id)
link.delete()
Also, if I make the import of UserLinksArchive local then celery starts, but when it's time to run async_task I get the same error. As you can see, problem is probably with AbstractBaseUser import. But I have no idea how to fix this. Probably, extuser/models.py code will help.
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.conf import settings
from django.contrib.auth.models import AbstractBaseUser
from django.contrib.auth.models import PermissionsMixin
from django.contrib.auth.models import BaseUserManager
from django.core.validators import MinValueValidator, MinLengthValidator
from django.db import models
class MyUserManager(BaseUserManager):
def create_user(self, username, password=None, **extra_fields):
if not username:
raise ValueError(u'No user')
user = self.model(username=username)
if password:
user.set_password(password)
user.save(using=self._db)
return user
def create_superuser(self, username, password):
user = self.create_user(username, password)
user.is_admin = True
user.is_superuser = True
user.save(using=self._db)
return user
class ExtUser(AbstractBaseUser, PermissionsMixin):
username = models.CharField(
max_length=255,
null=False,
unique=True,
validators=[MinLengthValidator(6)]
)
balance = models.FloatField(
default=0.0,
validators=[MinValueValidator(0.0)]
)
is_active = models.BooleanField(
default=True
)
is_admin = models.BooleanField(
default=False
)
def get_full_name(self):
return self.username
@property
def is_staff(self):
return self.is_admin
def get_short_name(self):
return self.username
def __str__(self):
return self.username
USERNAME_FIELD = 'username'
REQUIRED_FIELDS = []
objects = MyUserManager()
class Meta:
db_table = 'auth_user'
LINKTYPE = (
('p', 'Password restore'),
('o', 'Other'),
)
class UserLinksArchive(models.Model):
user = models.ForeignKey(
settings.AUTH_USER_MODEL,
default=1
)
hash = models.CharField(
max_length=32,
default=None,
unique=True
)
created_at = models.DateTimeField(
auto_now_add=True
)
type = models.CharField(
max_length=1,
default='o',
choices=LINKTYPE
)
And my celery.py:
#application/celery.py
# coding=utf-8
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'application.settings')
app = Celery('application')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('application.celery_conf', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
and celery_conf.py:
#application/celery_conf.py
# CELERY SETTINGS ALARMA
from django.conf import settings
CELERY_IMPORTS = ("application.tasks", "pages.tasks")
CELERY_BROKER_BACKEND = 'redis'
CELERY_BROKER_URL = settings.REDIS_URL
CELERY_RESULT_BACKEND = settings.REDIS_URL
CELERY_TASK_RESULT_EXPIRES = 60 * 60 * 24
CELERY_TIMEZONE = 'Europe/Moscow'
I've put import inside the task but the problem as I wrote still remained.
So, after some hours exploring the problem, I've found the reason. In my case the problem was in one of the packages in my requirements. I even don't know how it appeared there, really, have no idea (maybe will know if something will fall of). Here is my requirements.txt
amqp==2.1.4
anyjson==0.3.3
billiard==3.5.0.2
celery==4.0.2
Django==1.10.5
django-categories==1.4.2
django-celery==3.1.17
django-celery-beat==1.0.1
django-mptt==0.8.7
django-multiupload==0.5.1
django-shell-plus==1.1.7
django-widget-tweaks==1.4.1
gunicorn==19.6.0
kombu==4.0.2
lorem-ipsum-generator==0.3
lxml==3.7.2
mysqlclient==1.3.9
oauthlib==2.0.0
printer==2.0.0 <==(this package)
PyJWT==1.4.2
python-openid==2.2.5
python-social-auth==0.2.21
pytz==2016.10
redis==2.10.5
requests==2.11.1
requests-oauthlib==0.7.0
six==1.10.0
transliterate==1.8.1
tzlocal==1.3
unicode-slugify==0.1.1
vine==1.1.3
I've found that I can't even import packages from other apps. Runserver fell down with error like this:
/env/local/lib/python2.7/site-packages/printer.py", line 8
print("\t",end='')
^
SyntaxError: invalid syntax
My python version is 2.7 and this doesn't work. It's my first time when I feel the difference between 2.7 and 3.x print
Finally, I've just removed this package.
Flight is normal so far.