Coverage for portality / tasks / read_news.py: 37%

1import feedparser 

2 

3from portality import models 

4from portality.background import BackgroundTask, BackgroundApi 

5from portality.core import app 

6from portality.tasks.helpers import background_helper 

7from portality.tasks.redis_huey import scheduled_short_queue as queue 

8from portality.lib import dates 

9 

10 

11class FeedError(Exception): 

12 pass 

13 

14 

15class ReadNewsBackgroundTask(BackgroundTask): 

16 

17 __action__ = "read_news" 

18 

19 def run(self): 

20 """ 

21 Execute the task as specified by the background_jon 

22 :return: 

23 """ 

24 read_feed() 

25 

26 def cleanup(self): 

27 """ 

28 Cleanup after a successful OR failed run of the task 

29 :return: 

30 """ 

31 pass 

32 

33 @classmethod 

34 def prepare(cls, username, **kwargs): 

35 """ 

36 Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob, 

37 or fail with a suitable exception 

38 

39 :param kwargs: arbitrary keyword arguments pertaining to this task type 

40 :return: a BackgroundJob instance representing this task 

41 """ 

42 

43 # first prepare a job record 

44 return background_helper.create_job(username, cls.__action__, 

45 queue_id=huey_helper.queue_id, ) 

46 

47 @classmethod 

48 def submit(cls, background_job): 

49 """ 

50 Submit the specified BackgroundJob to the background queue 

51 

52 :param background_job: the BackgroundJob instance 

53 :return: 

54 """ 

55 background_job.save() 

56 read_news.schedule(args=(background_job.id,), delay=app.config.get('HUEY_ASYNC_DELAY', 10)) 

57 # fixme: schedule() could raise a huey.exceptions.HueyException and not reach redis- would that be logged? 

58 

59 

60# TODO factor this into the object above, rather than sitting out here as a function (it was migrated 

61# here from another file) 

62def read_feed(): 

63 """~~NewsReader:Feature->News:ExternalService""" 

64 feed_url = app.config.get("BLOG_FEED_URL") 

65 if feed_url is None: 

66 raise FeedError("No BLOG_FEED_URL defined in settings") 

67 

68 f = feedparser.parse(feed_url) 

69 if f.bozo > 0: 

70 raise FeedError(f.bozo_exception) 

71 

72 for e in f.entries: 

73 save_entry(e) 

74 

75def save_entry(entry): 

76 news = None 

77 existing = models.News.by_remote_id(entry.id) 

78 if len(existing) > 1: 

79 raise FeedError("There is more than one object with this id in the index: " + entry.id) 

80 elif len(existing) == 1: 

81 news = existing[0] 

82 else: 

83 news = models.News() 

84 

85 alts = [l.get("href") for l in entry.links if l.get("rel") == "alternate"] 

86 if len(alts) == 0: 

87 raise FeedError("Unable to get url of post from link@rel=alternate") 

88 

89 news.remote_id = entry.id 

90 news.url = alts[0] 

91 news.title = entry.title 

92 news.updated = dates.format(dates.timestruct2datetime(entry.updated_parsed)) 

93 news.summary = entry.summary 

94 news.published = dates.format(dates.timestruct2datetime(entry.published_parsed)) 

95 

96 news.save() 

97 

98 

99huey_helper = ReadNewsBackgroundTask.create_huey_helper(queue) 

100 

101 

102@huey_helper.register_schedule 

103def scheduled_read_news(): 

104 user = app.config.get("SYSTEM_USERNAME") 

105 job = ReadNewsBackgroundTask.prepare(user) 

106 ReadNewsBackgroundTask.submit(job) 

107 

108 

109@huey_helper.register_execute(is_load_config=False) 

110def read_news(job_id): 

111 job = models.BackgroundJob.pull(job_id) 

112 task = ReadNewsBackgroundTask(job) 

113 BackgroundApi.execute(task)