Coverage for portality/tasks/helpers/background_helper.py: 87%

129 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1""" collections of wrapper function for helping you to create BackgroundTask 

2~~BackgroundTasks:Feature~~ 

3""" 

4import inspect 

5import pkgutil 

6from typing import Callable, Type, Iterable, Tuple 

7 

8from huey import RedisHuey 

9 

10import portality.tasks 

11from portality import models, constants 

12from portality.background import BackgroundApi, BackgroundTask 

13from portality.core import app 

14from portality.decorators import write_required 

15from portality.tasks.redis_huey import events_queue, scheduled_long_queue, scheduled_short_queue, configure, schedule 

16 

17TaskFactory = Callable[[models.BackgroundJob], BackgroundTask] 

18_queue_for_action = None 

19 

20 

21def get_queue_id_by_task_queue(task_queue: RedisHuey): 

22 if task_queue is None: 

23 return constants.BGJOB_QUEUE_ID_UNKNOWN 

24 elif task_queue.name == events_queue.name: 

25 return constants.BGJOB_QUEUE_ID_EVENTS 

26 elif task_queue.name == scheduled_long_queue.name: 

27 return constants.BGJOB_QUEUE_ID_SCHEDULED_LONG 

28 elif task_queue.name == scheduled_short_queue.name: 

29 return constants.BGJOB_QUEUE_ID_SCHEDULED_SHORT 

30 else: 

31 app.logger.warning(f'unknown task_queue[{task_queue}]') 

32 return constants.BGJOB_QUEUE_ID_UNKNOWN 

33 

34 

35def create_job(username, action, 

36 queue_id=constants.BGJOB_QUEUE_ID_UNKNOWN, 

37 task_queue: RedisHuey = None, 

38 params=None): 

39 """ Common way to create BackgroundJob 

40 """ 

41 job = models.BackgroundJob() 

42 job.user = username 

43 job.action = action 

44 if params is not None: 

45 job.params = params 

46 

47 if task_queue is not None: 

48 queue_id = get_queue_id_by_task_queue(task_queue) 

49 job.queue_id = queue_id 

50 return job 

51 

52 

53def submit_by_bg_task_type(background_task: Type[BackgroundTask], **prepare_kwargs): 

54 """ Common way for BackgroundTask register_schedule """ 

55 user = app.config.get("SYSTEM_USERNAME") 

56 job = background_task.prepare(user, **prepare_kwargs) 

57 background_task.submit(job) 

58 

59 

60def execute_by_job_id(job_id, task_factory: TaskFactory): 

61 """ Common way to execute BackgroundTask by job_id """ 

62 job = models.BackgroundJob.pull(job_id) 

63 task = task_factory(job) 

64 BackgroundApi.execute(task) 

65 

66 

67def execute_by_bg_task_type(bg_task_type: Type[BackgroundTask], job_wrapper=None, **prepare_kwargs): 

68 """ wrapper for execute by BackgroundTask """ 

69 user = app.config.get("SYSTEM_USERNAME") 

70 job = bg_task_type.prepare(user, **prepare_kwargs) 

71 if job_wrapper is not None: 

72 job = job_wrapper(job) 

73 task = bg_task_type(job) 

74 BackgroundApi.execute(task) 

75 

76 return task 

77 

78 

79def register_execute(task_queue, task_name=None, script=True): 

80 """ 

81 decorator for register background job execute function 

82 """ 

83 

84 def wrapper(fn): 

85 if task_name: 

86 conf = configure(task_name) 

87 else: 

88 conf = {} 

89 

90 fn = write_required(script=script)(fn) 

91 try: 

92 fn = task_queue.task(**conf)(fn) 

93 except ValueError: 

94 # It's already registered - that's okay, we've probably accessed the _status endpoint and loaded the module 

95 return None 

96 return fn 

97 

98 return wrapper 

99 

100 

101class RedisHueyTaskHelper: 

102 """ 

103 some shortcut functions that help you implement functions that background job needed 

104 """ 

105 

106 def __init__(self, task_queue: RedisHuey, bgtask: Type[BackgroundTask]): 

107 self.task_queue = task_queue 

108 self.task_name = bgtask.__action__ 

109 self.task_factory = bgtask 

110 

111 @property 

112 def queue_id(self): 

113 return get_queue_id_by_task_queue(self.task_queue) 

114 

115 def register_schedule(self, fn): 

116 fn = write_required(script=True)(fn) 

117 try: 

118 fn = self.task_queue.periodic_task(schedule(self.task_name))(fn) 

119 except ValueError: 

120 # It's already registered - that's okay, we've probably accessed the _status endpoint and loaded the module 

121 return None 

122 return fn 

123 

124 def register_execute(self, is_load_config=False): 

125 return register_execute(self.task_queue, 

126 self.task_name if is_load_config else None) 

127 

128 def execute_common(self, job_id): 

129 """ Common way to execute BackgroundTask by job_id """ 

130 execute_by_job_id(job_id, self.task_factory) 

131 

132 def scheduled_common(self, **prepare_kwargs): 

133 """ Common way for BackgroundTask register_schedule """ 

134 submit_by_bg_task_type(self.task_factory, **prepare_kwargs) 

135 

136 

137def _get_background_task_spec(module): 

138 queue_id = None 

139 task_name = None 

140 bg_class = None 

141 for n, member in inspect.getmembers(module): 

142 if isinstance(member, RedisHuey): 

143 queue_id = get_queue_id_by_task_queue(member) 

144 elif ( 

145 inspect.isclass(member) 

146 and issubclass(member, BackgroundTask) 

147 and member != BackgroundTask 

148 ): 

149 task_name = getattr(member, '__action__', None) 

150 bg_class = member 

151 

152 if queue_id and task_name and bg_class: 

153 return queue_id, task_name, bg_class 

154 

155 return None 

156 

157 

158def lookup_queue_for_action(action): 

159 """ Find which queue an action is registered to, by action name """ 

160 """ Inspect the background tasks to find some useful details. Store in a singleton to reduce work. """ 

161 global _queue_for_action 

162 

163 if _queue_for_action is None: 

164 _queue_for_action = {_action: _queue for _queue, _action, _class in get_all_background_task_specs()} 

165 

166 return _queue_for_action.get(action, constants.BGJOB_QUEUE_ID_UNKNOWN) 

167 

168 

169def get_all_background_task_specs() -> Iterable[Tuple[str, str, Type]]: 

170 def _load_bgtask_safe(_mi): 

171 try: 

172 return _mi.module_finder.find_spec(_mi.name).loader.load_module(_mi.name) 

173 except RuntimeError as e: 

174 if 'No configuration for scheduled action' in str(e): 

175 app.logger.warning(f'config for {_mi.name} not found') 

176 return None 

177 raise e 

178 

179 module_infos = (m for m in pkgutil.walk_packages(portality.tasks.__path__) if not m.ispkg) 

180 modules = (_load_bgtask_safe(mi) for mi in module_infos) 

181 modules = filter(None, modules) 

182 bgspec_list = map(_get_background_task_spec, modules) 

183 bgspec_list = filter(None, bgspec_list) 

184 return bgspec_list 

185 

186 

187def get_value_safe(key, default_v, kwargs, default_cond_fn=None): 

188 """ get value from kwargs and return default_v if condition match 

189 """ 

190 v = kwargs.get(key, default_v) 

191 default_cond_fn = default_cond_fn or (lambda _v: _v is None) 

192 if default_cond_fn(v): 

193 v = default_v 

194 return v 

195 

196 

197def submit_by_background_job(background_job, execute_fn): 

198 """ Common way of `BackgroundTask.submit` 

199 """ 

200 background_job.save() 

201 execute_fn.schedule(args=(background_job.id,), delay=app.config.get('HUEY_ASYNC_DELAY', 10)) 

202 

203 

204def create_execute_fn(task_queue: RedisHuey, task_factory: TaskFactory, task_name=None, script=True): 

205 """ Common way to create execute_fn for BackgroundTask """ 

206 

207 @register_execute(task_queue, task_name=task_name, script=script) 

208 def _execute_fn(job_id): 

209 execute_by_job_id(job_id, task_factory) 

210 

211 return _execute_fn