Coverage for portality/models/uploads.py: 76%

1from portality.dao import DomainObject, ESMappingMissingError 

2from datetime import datetime 

3from copy import deepcopy 

4 

5 

6class FileUpload(DomainObject): 

7 __type__ = "upload" 

8 

9 @property 

10 def status(self): 

11 return self.data.get("status") 

12 

13 @property 

14 def local_filename(self): 

15 return self.id + ".xml" 

16 

17 @property 

18 def filename(self): 

19 return self.data.get("filename") 

20 

21 @property 

22 def schema(self): 

23 return self.data.get("schema") 

24 

25 @property 

26 def owner(self): 

27 return self.data.get("owner") 

28 

29 @property 

30 def imported(self): 

31 return self.data.get("imported", 0) 

32 

33 @property 

34 def failed_imports(self): 

35 return self.data.get("failed", 0) 

36 

37 @property 

38 def updates(self): 

39 return self.data.get("update", 0) 

40 

41 @property 

42 def new(self): 

43 return self.data.get("new", 0) 

44 

45 @property 

46 def error(self): 

47 return self.data.get("error") 

48 

49 @property 

50 def error_details(self): 

51 return self.data.get("error_details") 

52 

53 @property 

54 def failure_reasons(self): 

55 return self.data.get("failure_reasons", {}) 

56 

57 @property 

58 def created_timestamp(self): 

59 if "created_date" not in self.data: 

60 return None 

61 return datetime.strptime(self.data["created_date"], "%Y-%m-%dT%H:%M:%SZ") 

62 

63 def set_schema(self, s): 

64 self.data["schema"] = s 

65 

66 def upload(self, owner, filename, status="incoming"): 

67 self.data["filename"] = filename 

68 self.data["owner"] = owner 

69 self.data["status"] = status 

70 

71 def failed(self, message, details=None): 

72 self.data["status"] = "failed" 

73 self.data["error"] = message 

74 if details is not None: 

75 self.data["error_details"] = details 

76 

77 def validated(self, schema): 

78 self.data["status"] = "validated" 

79 self.data["schema"] = schema 

80 

81 def processed(self, count, update, new): 

82 self.data["status"] = "processed" 

83 self.data["imported"] = count 

84 self.data["update"] = update 

85 self.data["new"] = new 

86 

87 def partial(self, success, fail, update, new): 

88 self.data["status"] = "partial" 

89 self.data["imported"] = success 

90 self.data["failed"] = fail 

91 self.data["update"] = update 

92 self.data["new"] = new 

93 

94 def set_failure_reasons(self, shared, unowned, unmatched): 

95 self.data["failure_reasons"] = {} 

96 if shared is not None and len(shared) > 0: 

97 self.data["failure_reasons"]["shared"] = shared 

98 if unowned is not None and len(unowned) > 0: 

99 self.data["failure_reasons"]["unowned"] = unowned 

100 if unmatched is not None and len(unmatched) > 0: 

101 self.data["failure_reasons"]["unmatched"] = unmatched 

102 

103 def exists(self): 

104 self.data["status"] = "exists" 

105 

106 def downloaded(self): 

107 self.data["status"] = "downloaded" 

108 

109 @classmethod 

110 def list_valid(cls): 

111 q = ValidFileQuery() 

112 return cls.iterate(q=q.query(), page_size=10000) 

113 

114 @classmethod 

115 def list_remote(cls): 

116 q = ExistsFileQuery() 

117 return cls.iterate(q=q.query(), page_size=10000) 

118 

119 @classmethod 

120 def by_owner(cls, owner, size=10): 

121 q = OwnerFileQuery(owner) 

122 try: 

123 res = cls.query(q=q.query()) 

124 except ESMappingMissingError: 

125 return [] 

126 rs = [FileUpload(**r.get("_source")) for r in res.get("hits", {}).get("hits", [])] 

127 return rs 

128 

129 

130class ValidFileQuery(object): 

131 base_query = { 

132 "track_total_hits" : True, 

133 "query": { 

134 "term": {"status.exact": "validated"} 

135 }, 

136 "sort": [ 

137 {"created_date": "asc"} 

138 ] 

139 } 

140 

141 def __init__(self): 

142 self._query = deepcopy(self.base_query) 

143 

144 def query(self): 

145 return self._query 

146 

147 

148class ExistsFileQuery(object): 

149 base_query = { 

150 "track_total_hits" : True, 

151 "query": { 

152 "term": {"status.exact": "exists"} 

153 }, 

154 "sort": [ 

155 {"created_date": "asc"} 

156 ] 

157 } 

158 

159 def __init__(self): 

160 self._query = deepcopy(self.base_query) 

161 

162 def query(self): 

163 return self._query 

164 

165 

166class OwnerFileQuery(object): 

167 base_query = { 

168 "track_total_hits" : True, 

169 "query": { 

170 "bool": { 

171 "must": [] 

172 } 

173 }, 

174 "sort": [ 

175 {"created_date": "desc"} 

176 ], 

177 "size": 10 

178 } 

179 

180 def __init__(self, owner, size=10): 

181 self._query = deepcopy(self.base_query) 

182 owner_term = {"match": {"owner": owner}} 

183 self._query["query"]["bool"]["must"].append(owner_term) 

184 self._query["size"] = size 

185 

186 def query(self): 

187 return self._query