Coverage for portality / cms / build_fragments.py: 73%

1# ~~CMSBuildFragments:Script->CMS:Script~~ 

2import os 

3import random 

4import string 

5 

6import markdown 

7import shutil 

8import yaml 

9 

10from copy import deepcopy 

11from datetime import datetime 

12 

13from portality.cms import implied_attr_list 

14from portality.lib import paths 

15 

16BASE = "cms" 

17SRC = os.path.join(BASE, "pages") 

18OUT = os.path.join(BASE, "fragments") 

19FRONT_MATTER = os.path.join(BASE, "data/frontmatter.yml") 

20ERROR = os.path.join(BASE, "error_fragments.txt") 

21 

22 

23def _localise_paths(base_path=None): 

24 now = datetime.utcnow().timestamp() 

25 if base_path is None: 

26 return BASE, SRC, OUT, OUT + "." + str(now), FRONT_MATTER, FRONT_MATTER + "." + str(now), ERROR 

27 

28 return (os.path.join(base_path, BASE), 

29 os.path.join(base_path, SRC), 

30 os.path.join(base_path, OUT), 

31 os.path.join(base_path, OUT + "." + str(now)), 

32 os.path.join(base_path, FRONT_MATTER), 

33 os.path.join(base_path, FRONT_MATTER + "." + str(now)), 

34 os.path.join(base_path, ERROR)) 

35 

36 

37def _clear_tree(dir): 

38 if not os.path.exists(dir): 

39 os.makedirs(dir) 

40 for filename in os.listdir(dir): 

41 filepath = os.path.join(dir, filename) 

42 try: 

43 shutil.rmtree(filepath) 

44 except OSError: 

45 os.remove(filepath) 

46 

47 

48def create_random_str(n_char=10): 

49 s = string.ascii_letters + string.digits 

50 return ''.join(random.choices(s, k=n_char)) 

51 

52 

53def _swap(old, new): 

54 def _rm_dir_if_exist(_dir): 

55 if os.path.exists(_dir): 

56 try: 

57 shutil.rmtree(_dir) 

58 except OSError: 

59 os.remove(_dir) 

60 

61 tmp_old_dir = f'{old}.old.{create_random_str(20)}' 

62 if os.path.exists(old): 

63 _rm_dir_if_exist(tmp_old_dir) 

64 os.rename(old, tmp_old_dir) 

65 

66 os.rename(new, old) 

67 _rm_dir_if_exist(tmp_old_dir) 

68 

69 

70def build(base_path=None): 

71 #~~->CMSFragments:Build~~ 

72 base_dir, src_dir, out_dir, tmp_out_dir, fm_file, fm_tmp, error_file = None, None, None, None, None, None, None 

73 try: 

74 base_dir, src_dir, out_dir, tmp_out_dir, fm_file, fm_tmp, error_file = _localise_paths(base_path) 

75 _clear_tree(tmp_out_dir) 

76 if os.path.exists(error_file): 

77 os.remove(error_file) 

78 

79 extensions = [ 

80 "full_yaml_metadata", 

81 "toc", 

82 "markdown.extensions.tables", 

83 "markdown.extensions.fenced_code", 

84 'attr_list', 

85 'markdown_link_attr_modifier', 

86 "mdx_truly_sane_lists", 

87 "codehilite", 

88 implied_attr_list.ImpliedAttrListExtension() 

89 ] 

90 

91 cfg = { 

92 'markdown_link_attr_modifier': { 

93 "new_tab" : "external_only", 

94 "no_referrer" : "external_only" 

95 }, 

96 "codehilite" : { 

97 "css_class" : "highlight" 

98 } 

99 } 

100 

101 fm = {} 

102 

103 # Do all the page fragments 

104 for dirpath, dirnames, filenames in os.walk(src_dir): 

105 for fn in filenames: 

106 nfn = fn.rsplit(".", 1)[0] + ".html" 

107 sub_path = dirpath[len(src_dir) + 1:] 

108 file_ident = "/" + os.path.join(sub_path, nfn) 

109 outdir = os.path.join(tmp_out_dir, sub_path) 

110 out = os.path.join(outdir, nfn) 

111 input = os.path.join(src_dir, sub_path, fn) 

112 

113 with open(input) as f: 

114 md = markdown.Markdown(extensions=extensions, extension_configs=cfg) 

115 body = md.convert(f.read()) 

116 

117 nm = deepcopy(md.Meta) 

118 if nm is None: 

119 nm = {} 

120 if nm.get("toc") is True: 

121 nm["toc_tokens"] = md.toc_tokens 

122 

123 nm["frag"] = file_ident 

124 fm[file_ident] = nm 

125 

126 if not os.path.exists(outdir): 

127 os.makedirs(outdir) 

128 with open(out, "w") as g: 

129 g.write(body) 

130 

131 with open(fm_tmp, "w") as f: 

132 f.write(yaml.dump(fm)) 

133 

134 # we've got to here, so we are ready to shift everything into live position 

135 _swap(out_dir, tmp_out_dir) 

136 _swap(fm_file, fm_tmp) 

137 

138 except Exception as e: 

139 try: 

140 print("Error occurred building static pages") 

141 if error_file: 

142 with open(error_file, "w") as f: 

143 f.write(str(e)) 

144 

145 if tmp_out_dir and os.path.exists(tmp_out_dir): 

146 shutil.rmtree(tmp_out_dir) 

147 

148 if out_dir and os.path.exists(out_dir + ".old"): 

149 os.rename(out_dir + ".old", out_dir) 

150 

151 if fm_tmp and os.path.exists(fm_tmp): 

152 os.remove(fm_tmp) 

153 

154 if fm_file and os.path.exists(fm_file + ".old"): 

155 os.rename(fm_file + ".old", fm_file) 

156 

157 raise e 

158 

159 except: 

160 print("Error occurred building static pages and could not complete cleanup") 

161 raise e 

162 

163 

164if __name__ == "__main__": 

165 build(paths.get_project_root().as_posix())