Coverage for portality/tasks/helpers/background_helper.py: 87%
129 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-20 13:43 +0000
1""" collections of wrapper function for helping you to create BackgroundTask
2~~BackgroundTasks:Feature~~
3"""
4import inspect
5import pkgutil
6from typing import Callable, Type, Iterable, Tuple
8from huey import RedisHuey
10import portality.tasks
11from portality import models, constants
12from portality.background import BackgroundApi, BackgroundTask
13from portality.core import app
14from portality.decorators import write_required
15from portality.tasks.redis_huey import events_queue, scheduled_long_queue, scheduled_short_queue, configure, schedule
17TaskFactory = Callable[[models.BackgroundJob], BackgroundTask]
18_queue_for_action = None
21def get_queue_id_by_task_queue(task_queue: RedisHuey):
22 if task_queue is None:
23 return constants.BGJOB_QUEUE_ID_UNKNOWN
24 elif task_queue.name == events_queue.name:
25 return constants.BGJOB_QUEUE_ID_EVENTS
26 elif task_queue.name == scheduled_long_queue.name:
27 return constants.BGJOB_QUEUE_ID_SCHEDULED_LONG
28 elif task_queue.name == scheduled_short_queue.name:
29 return constants.BGJOB_QUEUE_ID_SCHEDULED_SHORT
30 else:
31 app.logger.warning(f'unknown task_queue[{task_queue}]')
32 return constants.BGJOB_QUEUE_ID_UNKNOWN
35def create_job(username, action,
36 queue_id=constants.BGJOB_QUEUE_ID_UNKNOWN,
37 task_queue: RedisHuey = None,
38 params=None):
39 """ Common way to create BackgroundJob
40 """
41 job = models.BackgroundJob()
42 job.user = username
43 job.action = action
44 if params is not None:
45 job.params = params
47 if task_queue is not None:
48 queue_id = get_queue_id_by_task_queue(task_queue)
49 job.queue_id = queue_id
50 return job
53def submit_by_bg_task_type(background_task: Type[BackgroundTask], **prepare_kwargs):
54 """ Common way for BackgroundTask register_schedule """
55 user = app.config.get("SYSTEM_USERNAME")
56 job = background_task.prepare(user, **prepare_kwargs)
57 background_task.submit(job)
60def execute_by_job_id(job_id, task_factory: TaskFactory):
61 """ Common way to execute BackgroundTask by job_id """
62 job = models.BackgroundJob.pull(job_id)
63 task = task_factory(job)
64 BackgroundApi.execute(task)
67def execute_by_bg_task_type(bg_task_type: Type[BackgroundTask], job_wrapper=None, **prepare_kwargs):
68 """ wrapper for execute by BackgroundTask """
69 user = app.config.get("SYSTEM_USERNAME")
70 job = bg_task_type.prepare(user, **prepare_kwargs)
71 if job_wrapper is not None:
72 job = job_wrapper(job)
73 task = bg_task_type(job)
74 BackgroundApi.execute(task)
76 return task
79def register_execute(task_queue, task_name=None, script=True):
80 """
81 decorator for register background job execute function
82 """
84 def wrapper(fn):
85 if task_name:
86 conf = configure(task_name)
87 else:
88 conf = {}
90 fn = write_required(script=script)(fn)
91 try:
92 fn = task_queue.task(**conf)(fn)
93 except ValueError:
94 # It's already registered - that's okay, we've probably accessed the _status endpoint and loaded the module
95 return None
96 return fn
98 return wrapper
101class RedisHueyTaskHelper:
102 """
103 some shortcut functions that help you implement functions that background job needed
104 """
106 def __init__(self, task_queue: RedisHuey, bgtask: Type[BackgroundTask]):
107 self.task_queue = task_queue
108 self.task_name = bgtask.__action__
109 self.task_factory = bgtask
111 @property
112 def queue_id(self):
113 return get_queue_id_by_task_queue(self.task_queue)
115 def register_schedule(self, fn):
116 fn = write_required(script=True)(fn)
117 try:
118 fn = self.task_queue.periodic_task(schedule(self.task_name))(fn)
119 except ValueError:
120 # It's already registered - that's okay, we've probably accessed the _status endpoint and loaded the module
121 return None
122 return fn
124 def register_execute(self, is_load_config=False):
125 return register_execute(self.task_queue,
126 self.task_name if is_load_config else None)
128 def execute_common(self, job_id):
129 """ Common way to execute BackgroundTask by job_id """
130 execute_by_job_id(job_id, self.task_factory)
132 def scheduled_common(self, **prepare_kwargs):
133 """ Common way for BackgroundTask register_schedule """
134 submit_by_bg_task_type(self.task_factory, **prepare_kwargs)
137def _get_background_task_spec(module):
138 queue_id = None
139 task_name = None
140 bg_class = None
141 for n, member in inspect.getmembers(module):
142 if isinstance(member, RedisHuey):
143 queue_id = get_queue_id_by_task_queue(member)
144 elif (
145 inspect.isclass(member)
146 and issubclass(member, BackgroundTask)
147 and member != BackgroundTask
148 ):
149 task_name = getattr(member, '__action__', None)
150 bg_class = member
152 if queue_id and task_name and bg_class:
153 return queue_id, task_name, bg_class
155 return None
158def lookup_queue_for_action(action):
159 """ Find which queue an action is registered to, by action name """
160 """ Inspect the background tasks to find some useful details. Store in a singleton to reduce work. """
161 global _queue_for_action
163 if _queue_for_action is None:
164 _queue_for_action = {_action: _queue for _queue, _action, _class in get_all_background_task_specs()}
166 return _queue_for_action.get(action, constants.BGJOB_QUEUE_ID_UNKNOWN)
169def get_all_background_task_specs() -> Iterable[Tuple[str, str, Type]]:
170 def _load_bgtask_safe(_mi):
171 try:
172 return _mi.module_finder.find_spec(_mi.name).loader.load_module(_mi.name)
173 except RuntimeError as e:
174 if 'No configuration for scheduled action' in str(e):
175 app.logger.warning(f'config for {_mi.name} not found')
176 return None
177 raise e
179 module_infos = (m for m in pkgutil.walk_packages(portality.tasks.__path__) if not m.ispkg)
180 modules = (_load_bgtask_safe(mi) for mi in module_infos)
181 modules = filter(None, modules)
182 bgspec_list = map(_get_background_task_spec, modules)
183 bgspec_list = filter(None, bgspec_list)
184 return bgspec_list
187def get_value_safe(key, default_v, kwargs, default_cond_fn=None):
188 """ get value from kwargs and return default_v if condition match
189 """
190 v = kwargs.get(key, default_v)
191 default_cond_fn = default_cond_fn or (lambda _v: _v is None)
192 if default_cond_fn(v):
193 v = default_v
194 return v
197def submit_by_background_job(background_job, execute_fn):
198 """ Common way of `BackgroundTask.submit`
199 """
200 background_job.save()
201 execute_fn.schedule(args=(background_job.id,), delay=app.config.get('HUEY_ASYNC_DELAY', 10))
204def create_execute_fn(task_queue: RedisHuey, task_factory: TaskFactory, task_name=None, script=True):
205 """ Common way to create execute_fn for BackgroundTask """
207 @register_execute(task_queue, task_name=task_name, script=script)
208 def _execute_fn(job_id):
209 execute_by_job_id(job_id, task_factory)
211 return _execute_fn