Coverage for portality/bll/services/background_task_status.py: 88%

110 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1""" 

2~~BackgroundTask:Monitoring~~ 

3""" 

4import itertools 

5from typing import Iterable, Optional 

6 

7from portality import constants 

8from portality.core import app 

9from portality.lib import dates 

10from portality.models.background import BackgroundJobQueryBuilder, BackgroundJob, SimpleBgjobQueue, \ 

11 LastCompletedJobQuery 

12from portality.tasks.helpers import background_helper 

13 

14 

15class BackgroundTaskStatusService: 

16 """ 

17 "background_task_status" is concept for list current background task status for monitoring. 

18 `create_background_status` is important function in service for create background_status data 

19 """ 

20 

21 @staticmethod 

22 def is_stable(val): 

23 return val == constants.BG_STATUS_STABLE 

24 

25 @staticmethod 

26 def to_bg_status_str(stable_val: bool) -> str: 

27 return constants.BG_STATUS_STABLE if stable_val else constants.BG_STATUS_UNSTABLE 

28 

29 def all_stable(self, items: Iterable, field_name='status') -> bool: 

30 return all(self.is_stable(q.get(field_name)) for q in items) 

31 

32 def all_stable_str(self, items: Iterable, field_name='status') -> str: 

33 return self.to_bg_status_str(self.all_stable(items, field_name)) 

34 

35 def create_last_successfully_run_status(self, action, last_run_successful_in=0, **_) -> dict: 

36 if last_run_successful_in == 0: 

37 return dict( 

38 status=constants.BG_STATUS_STABLE, 

39 last_run=None, 

40 last_run_status=None, 

41 err_msgs=[] 

42 ) 

43 

44 lr_query = (BackgroundJobQueryBuilder().action(action) 

45 .since(dates.before_now(last_run_successful_in)) 

46 .status_includes([constants.BGJOB_STATUS_COMPLETE, constants.BGJOB_STATUS_ERROR]) 

47 .size(1) 

48 .order_by('created_date', 'desc') 

49 .build_query_dict()) 

50 

51 try: 

52 lr_results = BackgroundJob.q2obj(q=lr_query) 

53 lr_job = lr_results[0] if len(lr_results) > 0 else None 

54 except Exception as e: 

55 app.logger.error(f'Error querying last successfully run status for action {action}: {e}') 

56 lr_job = None 

57 

58 status = constants.BG_STATUS_UNSTABLE 

59 lr = None 

60 last_run_status = None 

61 msg = ["No background jobs completed or errored in the time period"] 

62 

63 if lr_job is not None: 

64 lr = lr_job.created_date 

65 last_run_status = lr_job.status 

66 if lr_job.status == constants.BGJOB_STATUS_COMPLETE: 

67 status = constants.BG_STATUS_STABLE 

68 msg = [] 

69 else: 

70 msg = ["Last job did not complete successfully"] 

71 

72 return dict( 

73 status=status, 

74 last_run=lr, 

75 last_run_status=last_run_status, 

76 err_msgs=msg 

77 ) 

78 

79 def create_errors_status(self, action, check_sec=3600, allowed_num_err: Optional[int] = 0, **_) -> dict: 

80 if allowed_num_err is None: 

81 # error-count check disabled for this action - status is determined solely by 

82 # the last_run_successful check instead 

83 return dict( 

84 status=constants.BG_STATUS_STABLE, 

85 total=BackgroundJob.hit_count(query=SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_ERROR).query()), 

86 in_monitoring_period=None, 

87 err_msgs=[], 

88 ) 

89 

90 in_monitoring_query = SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_ERROR, since=dates.before_now(check_sec)) 

91 num_err_in_monitoring = BackgroundJob.hit_count(query=in_monitoring_query.query()) 

92 

93 # prepare errors messages 

94 err_msgs = [] 

95 if num_err_in_monitoring > allowed_num_err: 

96 err_msgs.append(f'too many error in monitoring period [{num_err_in_monitoring} > {allowed_num_err}]') 

97 

98 return dict( 

99 status=self.to_bg_status_str(not err_msgs), 

100 total=BackgroundJob.hit_count(query=SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_ERROR).query()), 

101 in_monitoring_period=num_err_in_monitoring, 

102 err_msgs=err_msgs, 

103 ) 

104 

105 def create_queued_status(self, action, total=2, oldest=1200, **_) -> dict: 

106 total_queued = BackgroundJob.hit_count(query=SimpleBgjobQueue(action, status=constants.BGJOB_STATUS_QUEUED).query()) 

107 oldest_query = (BackgroundJobQueryBuilder().action(action) 

108 .status_includes(constants.BGJOB_STATUS_QUEUED).size(1) 

109 .order_by('created_date', 'asc') 

110 .build_query_dict()) 

111 try: 

112 oldest_jobs = list(BackgroundJob.q2obj(q=oldest_query)) 

113 oldest_job = oldest_jobs and oldest_jobs[0] 

114 except Exception as e: 

115 app.logger.error(f'Error querying queued status for action {action}: {e}') 

116 oldest_jobs = [] 

117 oldest_job = None 

118 

119 err_msgs = [] 

120 limited_oldest_date = dates.before_now(oldest) 

121 if oldest_job and oldest_job.created_timestamp < limited_oldest_date: 

122 err_msgs.append('outdated queued job found[{}]. created_timestamp[{} < {}]'.format( 

123 oldest_job.id, 

124 oldest_job.created_timestamp, 

125 limited_oldest_date 

126 )) 

127 

128 if total_queued > total: 

129 err_msgs.append(f'too many queued job [{total_queued} > {total}]') 

130 

131 return dict( 

132 status=self.to_bg_status_str(not err_msgs), 

133 total=total_queued, 

134 oldest=oldest_job.created_date if oldest_job else None, 

135 err_msgs=err_msgs, 

136 ) 

137 

138 def create_queues_status(self, queue_name) -> dict: 

139 # define last_completed_job 

140 try: 

141 bgjob_list = BackgroundJob.q2obj(q=LastCompletedJobQuery(queue_name).query()) 

142 bgjob_list = list(bgjob_list) 

143 if bgjob_list: 

144 last_completed_date = bgjob_list[0].last_updated_timestamp 

145 else: 

146 last_completed_date = None 

147 except Exception as e: 

148 app.logger.error(f'Error querying last completed job for queue {queue_name}: {e}') 

149 bgjob_list = [] 

150 last_completed_date = None 

151 

152 errors = {action: self.create_errors_status(action, **config) for action, config 

153 in self.get_config_dict_by_queue_name('BG_MONITOR_ERRORS_CONFIG', queue_name).items()} 

154 

155 queued = {action: self.create_queued_status(action, **config) for action, config 

156 in self.get_config_dict_by_queue_name('BG_MONITOR_QUEUED_CONFIG', queue_name).items()} 

157 

158 last_run = {action: self.create_last_successfully_run_status(action, **config) for action, config 

159 in self.get_config_dict_by_queue_name('BG_MONITOR_LAST_SUCCESSFULLY_RUN_CONFIG', queue_name).items()} 

160 

161 # prepare for err_msgs 

162 limited_sec = app.config.get('BG_MONITOR_LAST_COMPLETED', {}).get(queue_name) 

163 if limited_sec is None: 

164 app.logger.warning(f'BG_MONITOR_LAST_COMPLETED for {queue_name} not found ') 

165 

166 err_msgs = [] 

167 if limited_sec is not None and last_completed_date: 

168 limited_date = dates.before_now(limited_sec) 

169 if last_completed_date < limited_date: 

170 err_msgs.append( 

171 f'last completed job is too old. [{last_completed_date} < {limited_date}]' 

172 ) 

173 

174 result_dict = dict( 

175 status=self.to_bg_status_str( 

176 not err_msgs and self.all_stable(itertools.chain(errors.values(), queued.values(), last_run.values()))), 

177 last_completed_job=last_completed_date and dates.format(last_completed_date), 

178 errors=errors, 

179 queued=queued, 

180 last_run_successful=last_run, 

181 err_msgs=err_msgs, 

182 ) 

183 return result_dict 

184 

185 @staticmethod 

186 def get_config_dict_by_queue_name(config_name, queue_name): 

187 bg_specs = background_helper.get_all_background_task_specs() 

188 actions = {action for qn, action, _ in bg_specs 

189 if qn == queue_name} 

190 

191 return { 

192 k: app.config.get(config_name, {}).get(k, app.config.get('BG_MONITOR_DEFAULT_CONFIG')) 

193 for k in actions 

194 } 

195 

196 def create_background_status(self) -> dict: 

197 queues = { 

198 queue_name: self.create_queues_status(queue_name) 

199 for queue_name in [constants.BGJOB_QUEUE_ID_EVENTS, 

200 constants.BGJOB_QUEUE_ID_SCHEDULED_LONG, 

201 constants.BGJOB_QUEUE_ID_SCHEDULED_SHORT] 

202 } 

203 

204 result_dict = dict( 

205 status=(self.all_stable_str(queues.values())), 

206 queues=queues, 

207 ) 

208 

209 # sort the results in the order of unstable status 

210 sorted_data = self.sort_dict_by_unstable_status(result_dict) 

211 

212 return sorted_data 

213 

214 def sort_dict_by_unstable_status(self, data): 

215 """ 

216 Sorts each dictionary within the nested structure by prioritizing items with 'status': 'unstable'. 

217 The overall structure of the input dictionary is preserved. 

218 """ 

219 if isinstance(data, dict): 

220 # Extract items with 'status': 'unstable' and other items 

221 unstable_items = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('status') == 'unstable'} 

222 other_items = {k: v for k, v in data.items() if k not in unstable_items} 

223 

224 # Recursively sort nested dictionaries 

225 for k in unstable_items: 

226 unstable_items[k] = self.sort_dict_by_unstable_status(unstable_items[k]) 

227 for k in other_items: 

228 other_items[k] = self.sort_dict_by_unstable_status(other_items[k]) 

229 

230 # Merge the dictionaries, with unstable items first 

231 return {**unstable_items, **other_items} 

232 else: 

233 # Return the item as is if it's not a dict 

234 return data