Coverage for portality/cms/build_fragments.py: 72%

1# ~~CMSBuildFragments:Script->CMS:Script~~ 

2import os 

3import random 

4import string 

5 

6import markdown 

7import shutil 

8import yaml 

9 

10from copy import deepcopy 

11from datetime import datetime 

12 

13from portality.cms import implied_attr_list 

14 

15BASE = "cms" 

16SRC = os.path.join(BASE, "pages") 

17OUT = os.path.join(BASE, "fragments") 

18FRONT_MATTER = os.path.join(BASE, "data/frontmatter.yml") 

19ERROR = os.path.join(BASE, "error_fragments.txt") 

20 

21 

22def _localise_paths(base_path=None): 

23 now = datetime.utcnow().timestamp() 

24 if base_path is None: 

25 return BASE, SRC, OUT, OUT + "." + str(now), FRONT_MATTER, FRONT_MATTER + "." + str(now), ERROR 

26 

27 return (os.path.join(base_path, BASE), 

28 os.path.join(base_path, SRC), 

29 os.path.join(base_path, OUT), 

30 os.path.join(base_path, OUT + "." + str(now)), 

31 os.path.join(base_path, FRONT_MATTER), 

32 os.path.join(base_path, FRONT_MATTER + "." + str(now)), 

33 os.path.join(base_path, ERROR)) 

34 

35 

36def _clear_tree(dir): 

37 if not os.path.exists(dir): 

38 os.makedirs(dir) 

39 for filename in os.listdir(dir): 

40 filepath = os.path.join(dir, filename) 

41 try: 

42 shutil.rmtree(filepath) 

43 except OSError: 

44 os.remove(filepath) 

45 

46 

47def create_random_str(n_char=10): 

48 s = string.ascii_letters + string.digits 

49 return ''.join(random.choices(s, k=n_char)) 

50 

51 

52def _swap(old, new): 

53 def _rm_dir_if_exist(_dir): 

54 if os.path.exists(_dir): 

55 try: 

56 shutil.rmtree(_dir) 

57 except OSError: 

58 os.remove(_dir) 

59 

60 tmp_old_dir = f'{old}.old.{create_random_str(20)}' 

61 if os.path.exists(old): 

62 _rm_dir_if_exist(tmp_old_dir) 

63 os.rename(old, tmp_old_dir) 

64 

65 os.rename(new, old) 

66 _rm_dir_if_exist(tmp_old_dir) 

67 

68 

69def build(base_path=None): 

70 #~~->CMSFragments:Build~~ 

71 base_dir, src_dir, out_dir, tmp_out_dir, fm_file, fm_tmp, error_file = None, None, None, None, None, None, None 

72 try: 

73 base_dir, src_dir, out_dir, tmp_out_dir, fm_file, fm_tmp, error_file = _localise_paths(base_path) 

74 _clear_tree(tmp_out_dir) 

75 if os.path.exists(error_file): 

76 os.remove(error_file) 

77 

78 extensions = [ 

79 "full_yaml_metadata", 

80 "toc", 

81 "markdown.extensions.tables", 

82 "markdown.extensions.fenced_code", 

83 'attr_list', 

84 'markdown_link_attr_modifier', 

85 "mdx_truly_sane_lists", 

86 "codehilite", 

87 implied_attr_list.ImpliedAttrListExtension() 

88 ] 

89 

90 cfg = { 

91 'markdown_link_attr_modifier': { 

92 "new_tab" : "external_only", 

93 "no_referrer" : "external_only" 

94 }, 

95 "codehilite" : { 

96 "css_class" : "highlight" 

97 } 

98 } 

99 

100 fm = {} 

101 

102 # Do all the page fragments 

103 for dirpath, dirnames, filenames in os.walk(src_dir): 

104 for fn in filenames: 

105 nfn = fn.rsplit(".", 1)[0] + ".html" 

106 sub_path = dirpath[len(src_dir) + 1:] 

107 file_ident = "/" + os.path.join(sub_path, nfn) 

108 outdir = os.path.join(tmp_out_dir, sub_path) 

109 out = os.path.join(outdir, nfn) 

110 input = os.path.join(src_dir, sub_path, fn) 

111 

112 with open(input) as f: 

113 md = markdown.Markdown(extensions=extensions, extension_configs=cfg) 

114 body = md.convert(f.read()) 

115 

116 nm = deepcopy(md.Meta) 

117 if nm is None: 

118 nm = {} 

119 if nm.get("toc") is True: 

120 nm["toc_tokens"] = md.toc_tokens 

121 

122 nm["frag"] = file_ident 

123 fm[file_ident] = nm 

124 

125 if not os.path.exists(outdir): 

126 os.makedirs(outdir) 

127 with open(out, "w") as g: 

128 g.write(body) 

129 

130 with open(fm_tmp, "w") as f: 

131 f.write(yaml.dump(fm)) 

132 

133 # we've got to here, so we are ready to shift everything into live position 

134 _swap(out_dir, tmp_out_dir) 

135 _swap(fm_file, fm_tmp) 

136 

137 except Exception as e: 

138 try: 

139 print("Error occurred building static pages") 

140 if error_file: 

141 with open(error_file, "w") as f: 

142 f.write(str(e)) 

143 

144 if tmp_out_dir and os.path.exists(tmp_out_dir): 

145 shutil.rmtree(tmp_out_dir) 

146 

147 if out_dir and os.path.exists(out_dir + ".old"): 

148 os.rename(out_dir + ".old", out_dir) 

149 

150 if fm_tmp and os.path.exists(fm_tmp): 

151 os.remove(fm_tmp) 

152 

153 if fm_file and os.path.exists(fm_file + ".old"): 

154 os.rename(fm_file + ".old", fm_file) 

155 

156 raise e 

157 

158 except: 

159 print("Error occurred building static pages and could not complete cleanup") 

160 raise e 

161 

162 

163if __name__ == "__main__": 

164 here = os.path.dirname(os.path.abspath(__file__)) 

165 base_path = os.path.dirname(os.path.dirname(here)) 

166 build(base_path)