CELERY 4.0.2 doesn't log into the file

Question

I'm using Celery (celery==4.0.2) in my Django project. The problem is that I can't make Celery log into the file. I've tried everything. The only thing which works is to specify the file in the command:

celery worker -A realestate_scanner -l info --purge --logfile=logs/celery.log

But the problem is that log can became very huge so I need to specify rotating logger and I want to run worker and beat as a deamons in production.

realestate_scanner/realestate_scanner/celery.py

from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'realestate_scanner.settings')
app = Celery('realestate_scanner')


app.config_from_object('django.conf:settings',)
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

realestate_scanner/realestate_scanner/settings.py ...

INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django_extensions',
'djmoney',
'storage',
'engineapp',
'presentation',
'import_export'
 ]


BROKER_URL = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Europe/Bratislava'
CELERY_LOG_FILE = 'test.log' # Not working
worker_log_file = 'test.log' # Not working

worker_hijack_root_logger = False

LOGGING = { # Not working
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'simple': {
            'format': '%(levelname)s %(message)s',
            'datefmt': '%y %b %d, %H:%M:%S',
        },
    },
    'handlers': {
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
        'celery': {
            'level': 'INFO',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': 'logs/celery.log',
            'formatter': 'simple',
            'maxBytes': 1024 * 1024 * 100,  # 100 mb
        },
        'scrapy': {
            'level': 'INFO',
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(BASE_DIR, 'logs/scrapy.log'),
            'formatter': 'simple'
        }
    },
    'loggers': {
        'celery': {
            'handlers': ['celery',],
            'level': 'INFO',
        },
        'scrapy': {
            'handlers': ['scrapy'],
            'level': 'INFO',
        }
    },

}

from logging.config import dictConfig

dictConfig(LOGGING)

Do you have any advices?

EDIT:

According to lapinkoira's answer, I've added this signal into celery.py file:

from __future__ import absolute_import
import os

import logging
from celery import Celery
from django.conf import settings

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'realestate_scanner.settings')
app = Celery('realestate_scanner')


app.config_from_object('django.conf:settings',)
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

import celery.signals

@celery.signals.setup_logging.connect
def on_celery_setup_logging(**kwargs):
    logger = logging.getLogger('realestate_scanner.realestate_scanner.celery')
    if not logger.handlers:
        handler = logging.FileHandler('celery_test_log.log')
        formatter = logging.Formatter(logging.BASIC_FORMAT)  # you may want to customize this.
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        logger.propagate = False

Signal probably works because there is celery_test_log.log file created but output goes to celery console again.

I'm probably not getting the logger I need but don't know how to get it.

Answer 1

mmm looks like celery dev says celery will still configure it's own loggers and ignore other configuration even if you disable hijack_root_logger

He recommends using celery logging signals:

source https://github.com/celery/celery/issues/3428

import celery.signals

@celery.signals.setup_logging.connect
def on_celery_setup_logging(**kwargs):
    pass