Coverage for portality / tasks / helpers / articles_upload_helper.py: 94%

1import os 

2import shutil 

3import traceback 

4from typing import List, Callable 

5 

6from portality import models 

7from portality.bll import DOAJ 

8from portality.bll.exceptions import IngestException, DuplicateArticleException, ArticleNotAcceptable 

9from portality.core import app 

10from portality.crosswalks.exceptions import CrosswalkException 

11from portality.models.uploads import BaseArticlesUpload 

12 

13 

14def file_failed(path): 

15 filename = os.path.split(path)[1] 

16 fad = app.config.get("FAILED_ARTICLE_DIR") 

17 dest = os.path.join(fad, filename) 

18 shutil.move(path, dest) 

19 

20 

21def mv_failed_file(path, job: models.BackgroundJob): 

22 try: 

23 file_failed(path) 

24 return True 

25 except: 

26 job.add_audit_message("Error cleaning up file which caused Exception: {x}" 

27 .format(x=traceback.format_exc())) 

28 return False 

29 

30 

31def upload_process(articles_upload: BaseArticlesUpload, 

32 job: models.BackgroundJob, 

33 articles_path, 

34 articles_factory: Callable[[str], List[models.Article]], ): 

35 """ 

36 save articles and handle exception and status 

37 

38 this function will update status `articles_upload` and `job`, but NOT save them. 

39 

40 :param articles_upload: 

41 :param job: 

42 :param articles_path: 

43 :param articles_factory: 

44 :return: 

45 """ 

46 

47 ingest_exception = False 

48 result = {} 

49 articles = [] 

50 try: 

51 articles = articles_factory(articles_path) 

52 account = models.Account.pull(articles_upload.owner) 

53 result = DOAJ.articleService().batch_create_articles(articles, account, add_journal_info=True) 

54 except (IngestException, CrosswalkException, ArticleNotAcceptable) as e: 

55 if hasattr(e, 'inner_message'): 

56 msg = "{exception}: {msg}. Inner message: {inner}. Stack: {x}".format( 

57 exception=e.__class__.__name__, msg=e.message, inner=e.inner_message, x=e.trace()) 

58 upload_detail = e.inner_message 

59 else: 

60 msg = "{exception}: {msg}.".format(exception=e.__class__.__name__, msg=e.message) 

61 upload_detail = None 

62 

63 result = e.result 

64 mark_fail_status(job, articles_upload, msg=msg, upload_msg=e.message, upload_detail=upload_detail) 

65 if mv_failed_file(articles_path, job): 

66 ingest_exception = True 

67 

68 except DuplicateArticleException as e: 

69 mark_fail_status(job, articles_upload, msg=str(e)) 

70 if not mv_failed_file(articles_path, job): 

71 return 

72 

73 except Exception as e: 

74 mark_fail_status(job, articles_upload, 

75 msg="Unanticipated error: {x}".format(x=traceback.format_exc()), 

76 upload_msg="Unanticipated error when importing articles") 

77 if not mv_failed_file(articles_path, job): 

78 return 

79 

80 success = result.get("success", 0) 

81 fail = result.get("fail", 0) 

82 update = result.get("update", 0) 

83 new = result.get("new", 0) 

84 shared = result.get("shared", []) 

85 unowned = result.get("unowned", []) 

86 unmatched = result.get("unmatched", []) 

87 

88 if success == 0 and fail > 0 and not ingest_exception: 

89 articles_upload.failed("All articles in file failed to import") 

90 job.add_audit_message("All articles in file failed to import") 

91 job.outcome_fail() 

92 if success > 0 and fail == 0: 

93 articles_upload.processed(success, update, new) 

94 if success > 0 and fail > 0: 

95 articles_upload.partial(success, fail, update, new) 

96 job.add_audit_message("Some articles in file failed to import correctly, so no articles imported") 

97 

98 articles_upload.set_failure_reasons(list(shared), list(unowned), list(unmatched)) 

99 job.add_audit_message("Shared ISSNs: " + ", ".join(list(shared))) 

100 job.add_audit_message("Unowned ISSNs: " + ", ".join(list(unowned))) 

101 job.add_audit_message("Unmatched ISSNs: " + ", ".join(list(unmatched))) 

102 

103 if new: 

104 ids = [a.id for a in articles] 

105 job.add_audit_message("Created/updated articles: " + ", ".join(list(ids))) 

106 

107 if not ingest_exception: 

108 try: 

109 os.remove(articles_path) # just remove the file, no need to keep it 

110 except Exception as e: 

111 job.add_audit_message(u"Error while deleting file {x}: {y}".format(x=articles_path, y=str(e))) 

112 

113 

114def mark_fail_status(job: models.BackgroundJob, file_upload: BaseArticlesUpload, 

115 msg: str, upload_msg: str = None, upload_detail: str = None): 

116 upload_msg = upload_msg or msg 

117 job.add_audit_message(msg) 

118 job.outcome_fail() 

119 file_upload.failed(upload_msg, details=upload_detail)