Django Celery IntegrityError

I want to create a progress bar for my project. I have a class and this class has some functions. Especially, one of them takes a long time (def download_all) and this is my main reason for wanting to create a progress bar.

I successfully set up celery, celery-progress, etc. and they work all fine. My problem is this: I want to integrate the progress bar to download_all function. I

It gives an error: IntegrityError at /o.../k... NOT NULL constraint failed: django_celery_results_taskresult.task_id

How can I solve it? functions.py

class myClass():

    def __init__(self, n_user, n_password, n_url, n_port, db_password, username):
        ...
        self.download_all(n_user, n_password, n_url, n_port, db_password)
        ...

    @shared_task(bind=True, name="my_add")
    def download_all(n_user, n_password, n_url, n_port, db_password)
     ...
     len_scans = len(scans)
     progress_recorder = ProgressRecorder(self)
     for s in scans:
         i = 0
         progress_recorder.set_progress(i + 1, len_scans)
         i += 1

views.py

def setup_wizard(request):
  ...
  functions.Zafiyet(setup.n_username, setup.n_password,
                                          setup.n_url, setup.n_port, setup.db_password,
                                          username=request.user.username)

traceback

Environment:


Request Method: POST
Request URL: http://127.0.0.1:8000/operasyonmerkezi/konfigurasyon

Django Version: 3.2.7
Python Version: 3.9.6
Installed Applications:
['django.contrib.admin',
 'django.contrib.auth',
 'django.contrib.contenttypes',
 'django.contrib.sessions',
 'django.contrib.messages',
 'django.contrib.staticfiles',
 'dashboard',
 'accounts',
 'logs',
 'crispy_forms',
 'django_apscheduler',
 'easy_timezones',
 'django_celery_results',
 'celery_progress']
Installed Middleware:
['django.middleware.security.SecurityMiddleware',
 'django.contrib.sessions.middleware.SessionMiddleware',
 'django.middleware.common.CommonMiddleware',
 'django.middleware.csrf.CsrfViewMiddleware',
 'django.contrib.auth.middleware.AuthenticationMiddleware',
 'django.contrib.messages.middleware.MessageMiddleware',
 'django.middleware.clickjacking.XFrameOptionsMiddleware']



Traceback (most recent call last):
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 581, in get_or_create
    return self.get(**kwargs), False
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 435, in get
    raise self.model.DoesNotExist(

During handling of the above exception (TaskResult matching query does not exist.), another exception occurred:
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\sqlite3\base.py", line 423, in execute
    return Database.Cursor.execute(self, query, params)

The above exception (NOT NULL constraint failed: django_celery_results_taskresult.task_id) was the direct cause of the following exception:
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
    response = get_response(request)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\contrib\auth\decorators.py", line 21, in _wrapped_view
    return view_func(request, *args, **kwargs)
  File "C:\Users\edeni\Desktop\hawkdragon\dashboard\views.py", line 128, in setup_wizard
    task = (functions.myClass(setup.n_username, setup.n_password,
  File "C:\Users\edeni\Desktop\hawkdragon\dashboard\functions.py", line 44, in __init__
    self.download_all(n_user, n_password, n_url, n_port, db_password)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\local.py", line 188, in __call__
    return self._get_current_object()(*a, **kw)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\app\task.py", line 389, in __call__
    return self.run(*args, **kwargs)
  File "C:\Users\edeni\Desktop\hawkdragon\dashboard\functions.py", line 153, in download_all
    progress_recorder.set_progress(i + 1, len_scans)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery_progress\backend.py", line 46, in set_progress
    self.task.update_state(
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\app\task.py", line 971, in update_state
    self.backend.store_result(
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\celery\backends\base.py", line 482, in store_result
    self._store_result(task_id, result, state, traceback,
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django_celery_results\backends\database.py", line 66, in _store_result
    self.TaskModel._default_manager.store_result(
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django_celery_results\managers.py", line 46, in _inner
    return fun(*args, **kwargs)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django_celery_results\managers.py", line 168, in store_result
    obj, created = self.using(using).get_or_create(task_id=task_id,
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 588, in get_or_create
    return self.create(**params), True
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 453, in create
    obj.save(force_insert=True, using=self.db)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 726, in save
    self.save_base(using=using, force_insert=force_insert,
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 763, in save_base
    updated = self._save_table(
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 868, in _save_table
    results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\base.py", line 906, in _do_insert
    return manager._insert(
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\query.py", line 1270, in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\models\sql\compiler.py", line 1416, in execute_sql
    cursor.execute(sql, params)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 98, in execute
    return super().execute(sql, params)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 66, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 75, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "C:\Users\edeni\Desktop\hawkdragon\myvenv\lib\site-packages\django\db\backends\sqlite3\base.py", line 423, in execute
    return Database.Cursor.execute(self, query, params)

Exception Type: IntegrityError at /operasyonmerkezi/konfigurasyon
Exception Value: NOT NULL constraint failed: django_celery_results_taskresult.task_id

Solution

If you want to check progress of task you have 3 options

store current progress as model field in db
store as key value in redis db
count at runtime by filtering and count element in db with changed status

Frontend can get this result by common REST API or websocket.

example minimum configs for REST API

model.py

class MyLongProcess(models.Model):
    active_uuid = models.UUIDField('Active process', null=True, blank=True)
    name = models.CharField('Name', max_length=255)
    current_step = models.IntegerField('Current step', default=0)
    total = models.IntegerField('Total', default=0)

    @property
    def percentage_sending(self):
        # or it can be computed by filtering elements processed in celery with complete status
        return int((current_step / total) * 100)

views.py

def initiate_execute_long_task(request, name):
    process = MyLongProcess.objects.create(active_uuid=uuid.uuid4(), name=name, total=BIG_COUNTED_VALUE)
    async_execute_long_task.delay(process)
    return HttpResponse()


from app.celery import app
@app.task
def async_execute_long_task(process)
    for i in range process.total:
        do_some_staff()
        process.current_step += 1
        process.save()

detail.html

<div class="progress-bar" role="progressbar"
  style="width: {{ my_model_object.percentage_sending }}%;"
  aria-valuenow="{{ my_model_object.percentage_sending }}"
  aria-valuemin="0" aria-valuemax="100">{{ my_model_object.percentage_sending }}%
</div>