Coverage for portality/background.py: 95%

132 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1import logging 

2import traceback 

3from copy import deepcopy 

4from typing import Iterable 

5from typing import TYPE_CHECKING 

6 

7from portality.constants import BgjobOutcomeStatus 

8 

9if TYPE_CHECKING: 

10 from portality.models import BackgroundJob 

11 

12from flask_login import login_user 

13from huey import RedisHuey 

14from huey.exceptions import RetryTask 

15 

16from portality import constants 

17from portality import models 

18from portality.bll import DOAJ 

19from portality.core import app 

20from portality import internationalise 

21 

22log = logging.getLogger(__name__) 

23 

24 

25class BackgroundException(Exception): 

26 pass 

27 

28 

29class BackgroundSummary(object): 

30 def __init__(self, job_id, affected=None, error=None): 

31 self.job_id = job_id 

32 self.affected = affected if affected is not None else {} 

33 self.error = error 

34 

35 def as_dict(self): 

36 return { 

37 "job_id": self.job_id, 

38 "affected": self.affected, 

39 "error": self.error 

40 } 

41 

42 

43class BackgroundApi(object): 

44 """ 

45 ~~BackgroundTasks:Feature~~ 

46 """ 

47 

48 @classmethod 

49 def execute(self, background_task: 'BackgroundTask'): 

50 # ~~->BackgroundTask:Process~~ 

51 # ~~->BackgroundJob:Model~~ 

52 job = background_task.background_job 

53 ctx = None 

54 acc = None 

55 if not hasattr(app, 'babel'): 

56 internationalise.internationalise(app) 

57 if job.user is not None: 

58 ctx = app.test_request_context("/") 

59 ctx.push() 

60 # ~~-> Account:Model~~ 

61 acc = models.Account.pull(job.user) # FIXME: what happens when this is the "system" user 

62 if acc is not None: 

63 login_user(acc) 

64 

65 job.start() 

66 job.add_audit_message("Job Started") 

67 job.save() 

68 

69 try: 

70 background_task.run() 

71 if job.outcome_status == BgjobOutcomeStatus.Pending: 

72 job.outcome_status = BgjobOutcomeStatus.Success 

73 except RetryTask: 

74 if job.reference is None: 

75 job.reference = {} 

76 retries = job.reference.get("retries", 0) 

77 job.reference["retries"] = retries + 1 

78 job.save() 

79 raise 

80 except Exception as e: 

81 log.error(f"Error in Background Task: {e}") 

82 job.fail() 

83 job.add_audit_message("Error in Job Run") 

84 job.add_audit_message("Caught in job runner during run: " + traceback.format_exc()) 

85 job.add_audit_message("Job Run Completed") 

86 

87 job.add_audit_message("Cleanup Started") 

88 try: 

89 background_task.cleanup() 

90 except Exception as e: 

91 job.fail() 

92 job.add_audit_message("Error in Cleanup Run") 

93 job.add_audit_message("Caught in job runner during cleanup: " + traceback.format_exc()) 

94 job.add_audit_message("Job Cleanup Completed") 

95 

96 job.add_audit_message("Job Finished") 

97 if not job.is_failed(): 

98 job.success() 

99 job.save() 

100 

101 # post-execution hook (may be overridden by task implementations) 

102 background_task.post_execute() 

103 

104 if ctx is not None: 

105 ctx.pop() 

106 

107 

108class BackgroundTask(object): 

109 """ 

110 All background tasks should extend from this object and override at least the following methods: 

111 

112 - run 

113 - cleanup 

114 - prepare (class method) 

115 

116 ~~BackgroundTask:Process~~ 

117 """ 

118 

119 __action__ = None 

120 """ static member variable defining the name of this task """ 

121 

122 def __init__(self, background_job: 'BackgroundJob'): 

123 self._background_job = background_job 

124 

125 @property 

126 def background_job(self) -> 'BackgroundJob': 

127 return self._background_job 

128 

129 def run(self): 

130 """ 

131 Execute the task as specified by the background_job 

132 :return: 

133 """ 

134 raise NotImplementedError() 

135 

136 def cleanup(self): 

137 """ 

138 Cleanup after a successful OR failed run of the task 

139 :return: 

140 """ 

141 raise NotImplementedError() 

142 

143 def post_execute(self): 

144 """ 

145 Hook executed after the job has been run and cleaned up. 

146 Default implementation triggers BACKGROUND_JOB_FINISHED event. 

147 Subclasses can override to change or extend the behaviour. 

148 """ 

149 job = self.background_job 

150 # trigger a status change event 

151 jdata = deepcopy(job.data) 

152 if "audit" in jdata: 

153 del jdata["audit"] 

154 eventsSvc = DOAJ.eventsService() 

155 eventsSvc.trigger(models.Event(constants.BACKGROUND_JOB_FINISHED, job.user, { 

156 "job": jdata 

157 })) 

158 

159 @classmethod 

160 def prepare(cls, username, **kwargs): 

161 """ 

162 Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, 

163 or fail with a suitable exception 

164 

165 :param username: the user creating the job 

166 :param kwargs: arbitrary keyword arguments pertaining to this task type 

167 :return: a BackgroundJob instance representing this task 

168 """ 

169 raise NotImplementedError() 

170 

171 @classmethod 

172 def submit(cls, background_job): 

173 """ 

174 Submit the specified BackgroundJob to the background queue 

175 

176 :param background_job: the BackgroundJob instance 

177 :return: 

178 """ 

179 raise NotImplementedError() 

180 

181 @classmethod 

182 def get_param(cls, params, param_name, default=None): 

183 return params.get('{}__{}'.format(cls.__action__, param_name), default) 

184 

185 @classmethod 

186 def set_param(cls, params, param_name, value): 

187 params['{}__{}'.format(cls.__action__, param_name)] = value 

188 

189 @classmethod 

190 def create_job_params(cls, **raw_param_dict: dict): 

191 new_param = {} 

192 for k, v in raw_param_dict.items(): 

193 cls.set_param(new_param, k, v) 

194 return new_param 

195 

196 def get_bgjob_params(self) -> dict: 

197 keys = self.background_job.params.keys() 

198 keys = (k.replace('{}__'.format(self.__action__), '') for k in keys) 

199 return {k: self.get_param(self.background_job.params, k) for k in keys} 

200 

201 @classmethod 

202 def set_reference(cls, refs, ref_name, value): 

203 refs['{}__{}'.format(cls.__action__, ref_name)] = value 

204 

205 @classmethod 

206 def create_huey_helper(cls, task_queue: RedisHuey): 

207 from portality.tasks.helpers import background_helper 

208 return background_helper.RedisHueyTaskHelper(task_queue, cls) 

209 

210 

211class AdminBackgroundTask(BackgroundTask): 

212 """~~AdminBackgroundTask:Process->BackgroundTask:Process~~""" 

213 

214 @classmethod 

215 def check_admin_privilege(cls, username): 

216 # ~~->Account:Model~~ 

217 a = models.Account.pull(username) 

218 if a is None: 

219 # very unlikely, as they would have had to log in to get to here.. 

220 raise BackgroundException('Admin account that is being used to prepare this job does not exist.') 

221 

222 if not a.has_role('admin'): 

223 raise BackgroundException('Account {} is not permitted to run this background task.'.format(username)) 

224 

225 @classmethod 

226 def prepare(cls, username, **kwargs): 

227 cls.check_admin_privilege(username)