1

I am running the setup that follows(just showing the beat setup for simplicity) for daemonizing my celery and beat workers on Elastic beanstalk. I am able daemonize the processes successfully however too many processes are being spawned.

Current Output

root     20409  0.7  9.1 473560 92452 ?        S    02:59   0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root     20412  0.6  7.8 388152 79228 ?        S    02:59   0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
root     20509  0.0  7.1 388748 72412 ?        S    02:59   0:00 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root     20585  0.6  7.7 387624 78340 ?        S    03:00   0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
root     20679  1.1  9.1 473560 92584 ?        S    03:01   0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root     20685  0.0  7.1 388768 72460 ?        S    03:01   0:00 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO

Desired output as achieved by after the environment deploys running kill -9 $(pgrep celery)

root     20794 20.6  7.7 387624 78276 ?        S    03:03   0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
root     20797 24.3  9.1 473560 92564 ?        S    03:03   0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root     20806  0.0  7.1 388656 72272 ?        S    03:03   0:00 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO

celeryBeat.sh

#!/usr/bin/env bash
      
/opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO

supervisor.conf

[unix_http_server]
file=/opt/python/run/supervisor.sock   ; (the path to the socket file)
;chmod=0700                 ; socket file mode (default 0700)
;chown=nobody:nogroup       ; socket file uid:gid owner

[supervisord]
logfile=/opt/python/log/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=10MB        ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10           ; (num of main logfile rotation backups;default 10)
loglevel=info                ; (log level;default info; others: debug,warn,trace)
pidfile=/opt/python/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
minfds=1024                  ; (min. avail startup file descriptors;default 1024)
minprocs=200                 ; (min. avail process descriptors;default 200)
directory=/opt/python/current/app    ; (default is not to cd during start)
;nocleanup=true              ; (don't clean up tempfiles at start;default false)

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///opt/python/run/supervisor.sock

[program:httpd]
command=/opt/python/bin/httpdlaunch
numprocs=1
directory=/opt/python/current/app
autostart=true
autorestart=unexpected
startsecs=1                   ; number of secs prog must stay running (def. 1)
startretries=3                ; max # of serial start failures (default 3)
exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
killasgroup=false             ; SIGKILL the UNIX process group (def false)
redirect_stderr=false
[include]
files: celery.conf

celery.conf

[program:beat:]
; Set full path to celery program if using virtualenv
command=sh /opt/python/etc/celeryBeat.sh
directory=/opt/python/current/app
; user=nobody
numprocs=1
stdout_logfile=/var/log/celery-beat.log
stderr_logfile=/var/log/celery-beat.log
autostart=true
autorestart=true
startsecs=60

; Need to wait for currently executing tasks to finish at shutdown.
; Increase this if you have very long running tasks.
stopwaitsecs = 60

; When resorting to send SIGKILL to the program to terminate it
; send SIGKILL to its whole process group instead,
; taking care of its children as well.
killasgroup=true

; if rabbitmq is supervised, set its priority higher
; so it starts first
priority=998

container commands

container_commands:
  01_celery_tasks:
    command: "cat .ebextensions/files/celery_configuration.txt > /opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh && chmod 744 /opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh"
    leader_only: true
  02_celery_tasks_run:
    command: "cat .ebextensions/files/beat_configuration.txt > /opt/python/etc/beat.sh && chmod 744 /opt/python/etc/celeryBeat.sh"
    leader_only: true 
 
  03_celery_tasks_run:
    command: "/opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh"
    leader_only: true

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.