90 lines
2.1 KiB
Python
90 lines
2.1 KiB
Python
|
import torch.multiprocessing as mp
|
||
|
|
mp.set_start_method("fork", force=True)
|
||
|
|
|
||
|
|
# from celery import Celery
|
||
|
|
|
||
|
|
# celery_app = Celery(
|
||
|
|
# "tasks",
|
||
|
|
# broker="redis://localhost:6379/0", # Redis as broker
|
||
|
|
# backend="redis://localhost:6379/0", # Redis for storing results
|
||
|
|
# )
|
||
|
|
|
||
|
|
# celery_app.conf.task_routes = {
|
||
|
|
# "tasks.*": {"queue": "default"},
|
||
|
|
# }
|
||
|
|
|
||
|
|
from celery import Celery
|
||
|
|
|
||
|
|
# app = Celery(
|
||
|
|
# "tasks",
|
||
|
|
# broker="redis://localhost:6379/0",
|
||
|
|
# backend="redis://localhost:6379/0",
|
||
|
|
# include=["tasks"]
|
||
|
|
# )
|
||
|
|
|
||
|
|
# app = Celery(
|
||
|
|
# "tasks",
|
||
|
|
# broker="pyamqp://guest@localhost//",
|
||
|
|
# backend="rpc://",
|
||
|
|
# include=["tasks"]
|
||
|
|
# )
|
||
|
|
|
||
|
|
app = Celery(
|
||
|
|
"celery_worker",
|
||
|
|
broker="pyamqp://guest@localhost//",
|
||
|
|
backend="rpc://",
|
||
|
|
)
|
||
|
|
|
||
|
|
# celery = Celery(
|
||
|
|
# "tasks",
|
||
|
|
# broker="pyamqp://guest@localhost//",
|
||
|
|
# backend="rpc://"
|
||
|
|
# )
|
||
|
|
|
||
|
|
app.conf.task_routes = {
|
||
|
|
"tasks.*": {"queue": "default"},
|
||
|
|
}
|
||
|
|
|
||
|
|
app.conf.worker_prefetch_multiplier = 1
|
||
|
|
app.conf.task_acks_late = True
|
||
|
|
|
||
|
|
|
||
|
|
# from celery import Celery
|
||
|
|
# from kombu import Queue
|
||
|
|
|
||
|
|
# celery = Celery(
|
||
|
|
# "tasks",
|
||
|
|
# broker="redis://localhost:6379/0",
|
||
|
|
# backend="redis://localhost:6379/0",
|
||
|
|
# include=["tasks"]
|
||
|
|
# )
|
||
|
|
|
||
|
|
# # Define task queues properly
|
||
|
|
# celery.conf.task_queues = (
|
||
|
|
# Queue("high_priority"),
|
||
|
|
# Queue("default"),
|
||
|
|
# Queue("low_priority"),
|
||
|
|
# )
|
||
|
|
|
||
|
|
# # Define task routing
|
||
|
|
# celery.conf.task_routes = {
|
||
|
|
# "tasks.text_query_task": {"queue": "high_priority"},
|
||
|
|
# "tasks.image_query_task": {"queue": "default"},
|
||
|
|
# "tasks.video_query_task": {"queue": "low_priority"},
|
||
|
|
# }
|
||
|
|
|
||
|
|
# # Define task rate limits
|
||
|
|
# celery.conf.task_annotations = {
|
||
|
|
# "tasks.text_query_task": {"rate_limit": "10/m"},
|
||
|
|
# "tasks.image_query_task": {"rate_limit": "5/m"},
|
||
|
|
# "tasks.video_query_task": {"rate_limit": "3/m"},
|
||
|
|
# }
|
||
|
|
|
||
|
|
# # Define task retries
|
||
|
|
# celery.conf.task_acks_late = True # Ensure task is only removed from queue when fully processed
|
||
|
|
# celery.conf.worker_prefetch_multiplier = 1 # Avoid one worker taking too many tasks at once
|
||
|
|
|
||
|
|
# # Define task time limits
|
||
|
|
# celery.conf.task_time_limit = 60 # 60 seconds max execution time
|
||
|
|
# celery.conf.task_soft_time_limit = 50 # Warn at 50 seconds
|