Im going to set periodic tasks with celery for what im trying to daemonize a django celery process in a test server using a script (referenced on the official website):
So, this is my celeryd file:
# Names of nodes to start
# most people will only start one node:
CELERYD_NODES="indicators"
# but you can also start multiple and configure settings
# for each in CELERYD_OPTS (see `celery multi --help` for examples):
#CELERYD_NODES="worker1 worker2 worker3"
# alternatively, you can specify the number of nodes to start:
#CELERYD_NODES=10
# Absolute or relative path to the 'celery' command:
#CELERY_BIN="/usr/local/bin/celery"
CELERY_BIN="/opt/pymis/envs/indicators/indicators_test/bin/celery"
# App instance to use
# comment out this line if you don't use an app
#CELERY_APP="sgiprocess"
# or fully qualified:
CELERY_APP="indicator.tasks:app"
# Where to chdir at start.
CELERYD_CHDIR="/opt/pymis/reps/indicator_repository/indicator_ms/"
# Extra command-line arguments to the worker
CELERYD_OPTS="--time-limit=300 --concurrency=8"
# %N will be replaced with the first part of the nodename.
CELERYD_LOG_FILE="/var/log/celery/worker1.log"
CELERYD_PID_FILE="/var/run/celery/worker1.pid"
# Workers should run as an unprivileged user.
# You need to create this user manually (or you can choose
# a user/group combination that already exists, e.g. nobody).
CELERYD_USER="celeryuser"
CELERYD_GROUP="mygroup"
# If enabled pid and log directories will be created if missing,
# and owned by the userid/group configured.
CELERY_CREATE_DIRS=1
This is my celerybeat file:
# Absolute or relative path to the 'celery' command:
#CELERY_BIN="/usr/local/bin/celery"
CELERY_BIN="/opt/pymis/envs/indicators/indicators_test/bin/celery"
# App instance to use
# comment out this line if you don't use an app
#CELERY_APP="sgiprocess"
# or fully qualified:
CELERY_APP="indicator.tasks:app"
# Where to chdir at start.
CELERYBEAT_CHDIR="/opt/pymis/reps/indicators_repository/indicator_ms/"
# Extra arguments to celerybeat
CELERYBEAT_OPTS="--schedule=/var/run/celery/celerybeat-schedule"
This is my error trace:
(indicators_test) user@server:/opt/pymis/reps/indicator_repository/indicator_ms$ sudo /etc/init.d/celeryd restart
celery init v10.1.
Using config script: /etc/default/celeryd
celery multi v4.0.2 (latentcall)
Traceback (most recent call last):
File "/usr/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/__main__.py", line 18, in <module>
main()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/__main__.py", line 14, in main
_main()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/celery.py", line 326, in main
cmd.execute_from_commandline(argv)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/celery.py", line 488, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 279, in execute_from_commandline
argv = self.setup_app_from_commandline(argv)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 481, in setup_app_from_commandline
self.app = self.find_app(app)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 503, in find_app
return find_app(app, symbol_by_name=self.symbol_by_name)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/app/utils.py", line 355, in find_app
sym = symbol_by_name(app, imp=imp)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 506, in symbol_by_name
return imports.symbol_by_name(name, imp=imp)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/kombu/utils/imports.py", line 56, in symbol_by_name
module = imp(module_name, package=package, **kwargs)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/importlib/__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 986, in _gcd_import
File "<frozen importlib._bootstrap>", line 969, in _find_and_load
File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 665, in exec_module
File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed
File "/opt/pymis/reps/indicator_repository/indicator_ms/indicator/tasks.py", line 7, in <module>
from .models import Value
File "/opt/pymis/reps/indicator_repository/indicator_ms/indicator/models.py", line 1, in <module>
from django.contrib.auth.models import User
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/contrib/auth/models.py", line 4, in <module>
from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/contrib/auth/base_user.py", line 52, in <module>
class AbstractBaseUser(models.Model):
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/db/models/base.py", line 105, in __new__
app_config = apps.get_containing_app_config(module)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/apps/registry.py", line 237, in get_containing_app_config
self.check_apps_ready()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/apps/registry.py", line 124, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
> Restarting node indicators@EMT-LIBREAPP: * Child terminated with exit code 1
FAILED
Thing is, when i manually initiate celery and celerybeat with:
celery -A config worker -l info
celery -A config beat
I don't get any error and everything loads well.
Update: This is my celery.py file
from __future__ import absolute_import
from celery import Celery
from django.conf import settings
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.test')
app = Celery('indicators')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
Update 2: This is my directory tree
indicator_ms (project root)
├── config
│ ├── settings
│ │ ├── __init__.py
│ │ ├── local.py
│ │ ├── test.py
│ │ └── production.py
│ ├── __init__.py
│ ├── celery.py
│ ├── urls.py
│ └── wsgi.py
│
└── indicators (app)
├── __init__.py
└── tasks.py
and this is the init.py file inside config folder (not settings folder)
from __future__ import absolute_import
from .celery import app as celery_app
As django celery tasks aren't properly a proyect module,we need to call the django.setup() method in our respective tasks modules.
In my case the solution was to set this lines inside task.py file before any other project related import:
import django
django.setup()