Coverage for portality / models / uploads.py: 72%

1from copy import deepcopy 

2 

3from portality.dao import DomainObject, ESMappingMissingError 

4from portality.lib import dates 

5 

6 

7class BaseArticlesUpload(DomainObject): 

8 """ 

9 Base class for article uploads. which handle status and error messages. 

10 

11 This is abstract class. it has no __type__ attribute. 

12 For object creation and query please use FileUpload or BulkArticles instead. 

13 

14 """ 

15 

16 @property 

17 def status(self): 

18 return self.data.get("status") 

19 

20 @property 

21 def owner(self): 

22 return self.data.get("owner") 

23 

24 @property 

25 def imported(self): 

26 return self.data.get("imported", 0) 

27 

28 @property 

29 def failed_imports(self): 

30 return self.data.get("failed", 0) 

31 

32 @property 

33 def updates(self): 

34 return self.data.get("update", 0) 

35 

36 @property 

37 def new(self): 

38 return self.data.get("new", 0) 

39 

40 @property 

41 def error(self): 

42 return self.data.get("error") 

43 

44 @property 

45 def error_details(self): 

46 return self.data.get("error_details") 

47 

48 @property 

49 def failure_reasons(self): 

50 return self.data.get("failure_reasons", {}) 

51 

52 @property 

53 def created_timestamp(self): 

54 if "created_date" not in self.data: 

55 return None 

56 return dates.parse(self.data["created_date"]) 

57 

58 def failed(self, message, details=None): 

59 self.data["status"] = "failed" 

60 self.data["error"] = message 

61 if details is not None: 

62 self.data["error_details"] = details 

63 

64 def validated(self): 

65 self.data["status"] = "validated" 

66 

67 def processed(self, count, update, new): 

68 self.data["status"] = "processed" 

69 self.data["imported"] = count 

70 self.data["update"] = update 

71 self.data["new"] = new 

72 

73 def partial(self, success, fail, update, new): 

74 self.data["status"] = "partial" 

75 self.data["imported"] = success 

76 self.data["failed"] = fail 

77 self.data["update"] = update 

78 self.data["new"] = new 

79 

80 def set_failure_reasons(self, shared, unowned, unmatched): 

81 self.data["failure_reasons"] = {} 

82 if shared is not None and len(shared) > 0: 

83 self.data["failure_reasons"]["shared"] = shared 

84 if unowned is not None and len(unowned) > 0: 

85 self.data["failure_reasons"]["unowned"] = unowned 

86 if unmatched is not None and len(unmatched) > 0: 

87 self.data["failure_reasons"]["unmatched"] = unmatched 

88 

89 def exists(self): 

90 self.data["status"] = "exists" 

91 

92 @classmethod 

93 def list_valid(cls): 

94 q = ValidFileQuery() 

95 return cls.iterate(q=q.query(), page_size=10000) 

96 

97 @classmethod 

98 def list_remote(cls): 

99 q = ExistsFileQuery() 

100 return cls.iterate(q=q.query(), page_size=10000) 

101 

102 @classmethod 

103 def by_owner(cls, owner, size=10): 

104 q = OwnerFileQuery(owner) 

105 try: 

106 res = cls.query(q=q.query()) 

107 except ESMappingMissingError: 

108 return [] 

109 rs = [cls(**r.get("_source")) for r in res.get("hits", {}).get("hits", [])] 

110 return rs 

111 

112 

113class FileUpload(BaseArticlesUpload): 

114 __type__ = "upload" 

115 

116 @classmethod 

117 def by_properties(cls, status=None, schema=None, max=10000): 

118 q = PropertiesQuery(status=status, schema=schema, size=max) 

119 return cls.object_query(q=q.query()) 

120 

121 @property 

122 def schema(self): 

123 return self.data.get("schema") 

124 

125 def set_schema(self, s): 

126 self.data["schema"] = s 

127 

128 @property 

129 def filename(self): 

130 return self.data.get("filename") 

131 

132 @property 

133 def local_filename(self): 

134 return self.id + ".xml" 

135 

136 def downloaded(self): 

137 self.data["status"] = "downloaded" 

138 

139 def validated(self, schema=None): 

140 self.data["status"] = "validated" 

141 if schema is not None: 

142 self.data["schema"] = schema 

143 

144 def upload(self, owner, filename, status="incoming"): 

145 self.data["filename"] = filename 

146 self.data["owner"] = owner 

147 self.data["status"] = status 

148 

149 

150class BulkArticles(BaseArticlesUpload): 

151 __type__ = "bulk_articles" 

152 

153 @property 

154 def local_filename(self): 

155 return self.id + ".json" 

156 

157 def incoming(self, owner): 

158 self.data["owner"] = owner 

159 self.data["status"] = "incoming" 

160 

161 

162class ValidFileQuery(object): 

163 base_query = { 

164 "track_total_hits": True, 

165 "query": { 

166 "term": {"status.exact": "validated"} 

167 }, 

168 "sort": [ 

169 {"created_date": "asc"} 

170 ] 

171 } 

172 

173 def __init__(self): 

174 self._query = deepcopy(self.base_query) 

175 

176 def query(self): 

177 return self._query 

178 

179 

180class ExistsFileQuery(object): 

181 base_query = { 

182 "track_total_hits": True, 

183 "query": { 

184 "term": {"status.exact": "exists"} 

185 }, 

186 "sort": [ 

187 {"created_date": "asc"} 

188 ] 

189 } 

190 

191 def __init__(self): 

192 self._query = deepcopy(self.base_query) 

193 

194 def query(self): 

195 return self._query 

196 

197 

198class OwnerFileQuery(object): 

199 base_query = { 

200 "track_total_hits": True, 

201 "query": { 

202 "bool": { 

203 "must": [] 

204 } 

205 }, 

206 "sort": [ 

207 {"created_date": "desc"} 

208 ], 

209 "size": 10 

210 } 

211 

212 def __init__(self, owner, size=10): 

213 self._query = deepcopy(self.base_query) 

214 owner_term = {"match": {"owner": owner}} 

215 self._query["query"]["bool"]["must"].append(owner_term) 

216 self._query["size"] = size 

217 

218 def query(self): 

219 return self._query 

220 

221class PropertiesQuery: 

222 def __init__(self, status=None, schema=None, size=10000): 

223 self._status = status 

224 self._schema = schema 

225 self._size = size 

226 

227 def query(self): 

228 query = { 

229 "query": { 

230 "bool": { 

231 "must": [] 

232 } 

233 } 

234 } 

235 

236 if self._status: 

237 query["query"]["bool"]["must"].append({"term": {"status.exact": self._status}}) 

238 

239 if self._schema: 

240 query["query"]["bool"]["must"].append({"term": {"schema.exact": self._schema}}) 

241 

242 if self._size: 

243 query["size"] = self._size 

244 

245 return query