Coverage for portality/tasks/anon_export.py: 88%

184 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-20 13:43 +0000

1import functools 

2import gzip 

3import os 

4import shutil 

5import uuid 

6from typing import Callable 

7from werkzeug.security import generate_password_hash 

8 

9from portality import models, dao 

10from portality.background import BackgroundTask 

11from portality.core import app, es_connection 

12from portality.lib import dates 

13from portality.lib.anon import basic_hash, anon_email 

14from portality.lib.dataobj import DataStructureException 

15from portality.lib.seamless import SeamlessException 

16from portality.models import Note 

17from portality.store import StoreFactory 

18from portality.tasks.helpers import background_helper 

19from portality.tasks.redis_huey import scheduled_long_queue as queue 

20 

21email_subs = {} 

22email_counter = 0 

23name_subs = {} 

24name_counter = 0 

25password = None 

26 

27def create_random_str(n_char=10): 

28 import string, random 

29 

30 s = string.ascii_letters + string.digits 

31 return ''.join(random.choices(s, k=n_char)) 

32 

33def _anonymise_email(record): 

34 if record.email not in email_subs: 

35 global email_counter 

36 email_counter += 1 

37 email_subs[record.email] = str(email_counter) + "@example.com" 

38 record.set_email(email_subs[record.email]) 

39 return record 

40 

41 

42def _anonymise_name(record): 

43 if record.name is None: 

44 return record 

45 if record.name not in name_subs: 

46 global name_counter 

47 name_counter += 1 

48 name_subs[record.name] = "User " + str(name_counter) 

49 record.set_name(name_subs[record.name]) 

50 return record 

51 

52 

53def _anonymise_admin(record): 

54 d = record.data 

55 if "admin" in d and "index" in d["admin"] and "notes" in d["admin"]["index"]: 

56 anon_notes = [] 

57 for note in record.data["admin"]["index"]["notes"]: 

58 anon_notes.append("---note removed for data security---") 

59 record.data["admin"]["index"]["notes"] = anon_notes 

60 

61 return record 

62 

63 

64def _reset_api_key(record): 

65 if record.api_key is not None: 

66 record.generate_api_key() 

67 return record 

68 

69 

70def _reset_password(record): 

71 global password 

72 if password is None: 

73 password = generate_password_hash(uuid.uuid4().hex) 

74 record.set_password_hash(password) 

75 return record 

76 

77 

78def _drop_unanonymisable(type_, record): 

79 # A record we can't safely construct/anonymise must never be exported raw - an "anon" export 

80 # containing un-anonymised data isn't anonymous. Drop it and log so the gap is visible. 

81 if not isinstance(record, dict): 

82 app.logger.warning(f"anon_export: dropping {type_} record - could not be anonymised") 

83 return None 

84 

85 app.logger.warning(f"anon_export: dropping {type_} record '{record.get('id')}' - could not be anonymised") 

86 return None 

87 

88 

89# transform functions - return the JSON data source since 

90# esprit doesn't understand our model classes 

91def anonymise_account(record): 

92 try: 

93 a = models.Account(**record) 

94 except DataStructureException: 

95 return _drop_unanonymisable("account", record) 

96 

97 a = _anonymise_email(a) 

98 a = _anonymise_name(a) 

99 a = _reset_api_key(a) 

100 a = _reset_password(a) 

101 

102 return a.data 

103 

104 

105def anonymise_journal(record): 

106 try: 

107 j = models.Journal(**record) 

108 except (DataStructureException, SeamlessException): 

109 return _drop_unanonymisable("journal", record) 

110 

111 return _anonymise_admin(j).data 

112 

113 

114def anonymise_application(record): 

115 try: 

116 appl = models.Application(**record) 

117 except (DataStructureException, SeamlessException): 

118 return _drop_unanonymisable("application", record) 

119 

120 appl = _anonymise_admin(appl) 

121 return appl.data 

122 

123 

124def anonymise_background_job(record): 

125 try: 

126 bgjob = models.BackgroundJob(**record) 

127 except DataStructureException: 

128 return _drop_unanonymisable("background_job", record) 

129 

130 if bgjob.params and 'suggestion_bulk_edit__note' in bgjob.params: 

131 bgjob.params['suggestion_bulk_edit__note'] = basic_hash(bgjob.params['suggestion_bulk_edit__note']) 

132 

133 return bgjob.data 

134 

135def anonymise_note(record): 

136 try: 

137 note = Note(**record) 

138 except (DataStructureException, SeamlessException): 

139 return _drop_unanonymisable("note", record) 

140 

141 note.note = "---note removed for data security---" 

142 note.author_id = create_random_str() 

143 return note.data 

144 

145 

146anonymisation_procedures = { 

147 'account': anonymise_account, 

148 'background_job': anonymise_background_job, 

149 'journal': anonymise_journal, 

150 'application': anonymise_application, 

151 "note": anonymise_note 

152} 

153 

154# types that should use prefix queries to optimise performance for bulk exporting 

155striped = { 

156 # "application": True, 

157 "article": True 

158} 

159 

160skip = app.config.get("ANON_EXPORT_SKIP_LIST",[]) 

161 

162 

163def _copy_on_complete(path, logger_fn, tmpStore, mainStore, container): 

164 name = os.path.basename(path) 

165 raw_size = os.path.getsize(path) 

166 logger_fn(("Compressing temporary file {x} (from {y} bytes)".format(x=path, y=raw_size))) 

167 zipped_name = name + ".gz" 

168 dir = os.path.dirname(path) 

169 zipped_path = os.path.join(dir, zipped_name) 

170 with open(path, "rb") as f_in, gzip.open(zipped_path, "wb") as f_out: 

171 shutil.copyfileobj(f_in, f_out) 

172 zipped_size = os.path.getsize(zipped_path) 

173 logger_fn(("Storing from temporary file {x} ({y} bytes)".format(x=zipped_name, y=zipped_size))) 

174 mainStore.store(container, name, source_path=zipped_path) 

175 tmpStore.delete_file(container, name) 

176 tmpStore.delete_file(container, zipped_name) 

177 

178 

179def run_anon_export(tmpStore, mainStore, container, clean=False, limit=None, batch_size=100000, 

180 logger_fn: Callable[[str], None] = None): 

181 if logger_fn is None: 

182 logger_fn = print 

183 if clean: 

184 mainStore.delete_container(container) 

185 

186 doaj_types = es_connection.indices.get(app.config['ELASTIC_SEARCH_DB_PREFIX'] + '*') 

187 true_names = [] 

188 for t, d in doaj_types.items(): 

189 if "aliases" in d: 

190 aliases = list(d["aliases"].keys()) 

191 if len(aliases) > 0: 

192 true_names.append(aliases[0]) 

193 else: 

194 true_names.append(t) 

195 else: 

196 true_names.append(t) 

197 

198 # aliases = es_connection.indices.get_alias(index=app.config['ELASTIC_SEARCH_DB_PREFIX'] + "*") 

199 # aliased_types = [item for sublist in [list(v.get("aliases").keys()) for k, v in aliases.items()] for item in sublist] 

200 # prefixed_types = list(set(doaj_types + aliased_types)) 

201 # type_list = [t[len(app.config['ELASTIC_SEARCH_DB_PREFIX']):] for t in doaj_types] + [a[len(app.config['ELASTIC_SEARCH_DB_PREFIX']):] for a in aliases] 

202 type_list = [a[len(app.config['ELASTIC_SEARCH_DB_PREFIX']):] for a in true_names] 

203 

204 for type_ in type_list: 

205 if type_ in skip: 

206 logger_fn("skipping " + type_) 

207 continue 

208 model = models.lookup_models_by_type(type_, dao.DomainObject) 

209 if not model: 

210 logger_fn("unable to locate model for " + type_) 

211 continue 

212 

213 filename = type_ + ".bulk" 

214 output_file = tmpStore.path(container, filename, create_container=True, must_exist=False) 

215 logger_fn((dates.now_str() + " " + type_ + " => " + output_file + ".*")) 

216 # iter_q = {"query": {"match_all": {}}, "sort": [{"_id": {"order": "asc"}}]} 

217 transform = None 

218 if type_ in anonymisation_procedures: 

219 transform = anonymisation_procedures[type_] 

220 

221 # Use the model's dump method to write out this type to file 

222 out_rollover_fn = functools.partial(_copy_on_complete, logger_fn=logger_fn, tmpStore=tmpStore, 

223 mainStore=mainStore, container=container) 

224 

225 s = striped.get(type_, False) 

226 _ = model.dump(limit=limit, transform=transform, out_template=output_file, out_batch_sizes=batch_size, 

227 out_rollover_callback=out_rollover_fn, es_bulk_fields=["_id"], striped=s, prefix_size=3, logger=logger_fn) 

228 

229 logger_fn((dates.now_str() + " done\n")) 

230 

231 tmpStore.delete_container(container) 

232 

233 

234class AnonExportBackgroundTask(BackgroundTask): 

235 """ 

236 ~~AnonExport:Feature->BackgroundTask:Process~~ 

237 """ 

238 __action__ = "anon_export" 

239 

240 def run(self): 

241 kwargs = self.get_bgjob_params() 

242 kwargs['logger_fn'] = self.background_job.add_audit_message 

243 

244 tmpStore = StoreFactory.tmp() 

245 mainStore = StoreFactory.get("anon_data") 

246 container = app.config.get("STORE_ANON_DATA_CONTAINER") 

247 

248 run_anon_export(tmpStore, mainStore, container, **kwargs) 

249 self.background_job.add_audit_message("Anon export completed") 

250 

251 def cleanup(self): 

252 pass 

253 

254 @classmethod 

255 def prepare(cls, username, **kwargs): 

256 params = {} 

257 cls.set_param(params, 'clean', background_helper.get_value_safe('clean', False, kwargs)) 

258 cls.set_param(params, "limit", kwargs.get('limit')) 

259 cls.set_param(params, "batch_size", background_helper.get_value_safe('batch_size', 100000, kwargs)) 

260 return background_helper.create_job(username=username, 

261 action=cls.__action__, 

262 params=params, 

263 queue_id=huey_helper.queue_id, ) 

264 

265 @classmethod 

266 def submit(cls, background_job): 

267 background_job.save() 

268 anon_export.schedule(args=(background_job.id,), delay=app.config.get('HUEY_ASYNC_DELAY', 10)) 

269 

270 

271huey_helper = AnonExportBackgroundTask.create_huey_helper(queue) 

272 

273 

274@huey_helper.register_schedule 

275def scheduled_anon_export(): 

276 background_helper.submit_by_bg_task_type(AnonExportBackgroundTask, 

277 clean=app.config.get("TASKS_ANON_EXPORT_CLEAN", False), 

278 limit=app.config.get("TASKS_ANON_EXPORT_LIMIT", None), 

279 batch_size=app.config.get("TASKS_ANON_EXPORT_BATCH_SIZE", 100000)) 

280 

281 

282@huey_helper.register_execute(is_load_config=False) 

283def anon_export(job_id): 

284 background_helper.execute_by_job_id(job_id, AnonExportBackgroundTask)