Coverage for src/hods/config/base.py: 100.00%

1"""hods - home directory synchronization. 

2 

3Copyright (C) 2016-2020 Mathias Stelzer <knoppo@rolln.de> 

4 

5hods is free software: you can redistribute it and/or modify 

6it under the terms of the GNU General Public License as published by 

7the Free Software Foundation, either version 3 of the License, or 

8(at your option) any later version. 

9 

10hods is distributed in the hope that it will be useful, 

11but WITHOUT ANY WARRANTY; without even the implied warranty of 

12MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

13GNU General Public License for more details. 

14 

15You should have received a copy of the GNU General Public License 

16along with this program. If not, see <http://www.gnu.org/licenses/>. 

17""" 

18import datetime 

19import json 

20import logging 

21import os 

22import subprocess 

23 

24from hods.node import validate_child_path 

25from hods.path import File 

26from hods.utils import clean_server, pw_home, run_rsync, run_ssh, which 

27 

28logger = logging.getLogger(__name__) 

29 

30 

31class JSONConfigMixin: 

32 """json configuration file.""" 

33 

34 def parse(self, raw): 

35 """Parse raw data to objects. 

36 

37 :param raw: `str` - Raw string 

38 :return: `dict` - Parsed dictionary 

39 """ 

40 return json.loads(raw) 

41 

42 def serialize(self, data): 

43 """Serialize objects to raw data. 

44 

45 :param data: `dict` - Data dictionary 

46 :return: `str` - Writable raw data 

47 """ 

48 return json.dumps(data) 

49 

50 def load(self): 

51 """Read, parse and return the file contents. 

52 

53 :return: `dict` - The parsed file content. 

54 """ 

55 try: 

56 raw = self.read() 

57 except IOError as e: 

58 if e.errno != 2: 

59 raise e 

60 return {} 

61 return self.parse(raw) 

62 

63 def save(self, data): 

64 """Serialize the given data and write it. 

65 

66 :param data: `dict` - The data to serialize and write 

67 """ 

68 self.write(self.serialize(data)) 

69 

70 

71class JSONAttributeConfigMixin(JSONConfigMixin): 

72 """json configuration file that stores its attributes.""" 

73 

74 # list of names of storage attributes 

75 attributes = [] 

76 

77 def get_attributes(self): 

78 """Return the list of names of storage attributes.""" 

79 return self.attributes 

80 

81 def parse(self, raw): 

82 """Parse the given raw string and call attribute parser's. 

83 

84 :param raw: `str` - Raw json string 

85 :return: `dict` - Parsed python dictionary 

86 """ 

87 data = super().parse(raw) 

88 for name in self.get_attributes(): 

89 parse = getattr(self, '_parse_{}'.format(name), None) 

90 if name not in data: 

91 continue 

92 if callable(parse): 

93 parse(data[name]) 

94 del data[name] 

95 return data 

96 

97 def serialize(self, data): 

98 """Call attribute serializer's. 

99 

100 :param data: `dict` - Python dictionary 

101 :return: `str` - Serialized json string 

102 """ 

103 for name in self.get_attributes(): 

104 serialize = getattr(self, '_serialize_{}'.format(name), None) 

105 if callable(serialize): 

106 data[name] = serialize() 

107 return super().serialize(data) 

108 

109 def set_data(self, data): 

110 """Assign all keys as attributes.""" 

111 for name in self.get_attributes(): 

112 default = getattr(self, name, None) 

113 setattr(self, name, data.pop(name, default)) 

114 

115 def get_data(self): 

116 """Collect all attributes and generate items.""" 

117 for name in self.get_attributes(): 

118 yield name, getattr(self, name, None) 

119 

120 def load(self): 

121 """Load the file and assign all keys as attributes. 

122 

123 :raises: `IOError` - File processing error 

124 """ 

125 self.set_data(super().load()) 

126 

127 def save(self): 

128 """Collect the attributes and store them.""" 

129 super().save(dict(self.get_data())) 

130 

131 

132DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' 

133 

134 

135class BaseConfigFile(JSONAttributeConfigMixin, File): 

136 """Base configuration file storing a source tree.""" 

137 

138 attributes = [ 

139 'tree', 

140 'variables', 

141 ] 

142 

143 def __init__(self, path): 

144 """Initialize configuration file.""" 

145 super().__init__(path) 

146 self.variables = {} 

147 

148 def _parse_tree(self, data): 

149 """Parse and set the given tree configuration.""" 

150 self.tree.load_data(data) 

151 

152 def _serialize_tree(self): 

153 """Serialize and return the tree attribute.""" 

154 return self.tree.as_dict() 

155 

156 

157class RemoteMkDirProcessError(subprocess.CalledProcessError): 

158 """Raised when subprocess for remote directory creation fails.""" 

159 

160 pass 

161 

162 

163class Settings(BaseConfigFile): 

164 """Local configuration of the current user/home.""" 

165 

166 def get_attributes(self): 

167 """Return the list of names of storage attributes.""" 

168 yield from super().get_attributes() 

169 yield 'installed_features' 

170 yield 'server' 

171 yield 'is_proxy' 

172 yield 'server_is_proxy' 

173 yield 'cascade' 

174 yield 'pulled_at' 

175 yield 'ssh_agent_key' 

176 yield 'ssh_agent_mode' 

177 

178 def __init__(self, load=True): 

179 """Initialize config file.""" 

180 from hods.config.tree import SourceTree 

181 from hods.version import __version__ 

182 

183 self.home_path = pw_home() 

184 self.hods_path = os.path.join(self.home_path, '.hods') 

185 

186 self.version = __version__ 

187 

188 path = os.path.join(self.hods_path, 'settings.json') 

189 super().__init__(path) 

190 

191 self.installed_features = [] 

192 self.server = None # empty for master 

193 self.is_proxy = False # sync all features (ignore installed_features in pull and push) 

194 self.server_is_proxy = False # use rsync to sync git repositories with server 

195 self.cascade = True 

196 self.ssh_agent_key = None 

197 self.ssh_agent_mode = 'ask' if which('ssh-agent') else 'warn' 

198 

199 self.pulled_at = None 

200 self.tree = SourceTree(self) 

201 

202 if load: 

203 self.load() 

204 

205 @property 

206 def is_master(self): 

207 """Return `True` if no server is configured.""" 

208 return not self.server 

209 

210 @property 

211 def is_server(self): 

212 """Return True if no server is configured or if this is a proxy.""" 

213 return self.is_master or self.is_proxy 

214 

215 def get_server_path(self, path): 

216 """Build and return the remote path (on the server) of the given file.""" 

217 server = clean_server(self.server) 

218 if server is None: 

219 return 

220 path = validate_child_path(path, self.home_path) 

221 return '{}:{}'.format(server, path) 

222 

223 def rmkdir(self, path): 

224 """Create the given path in the home directory on the configured server. 

225 

226 Returns: The completed process information or `None` if the directory exists. 

227 

228 Raises: 

229 ValueError: If the given path is absolute and does not start with the home directory. 

230 RemoteMkDirProcessError: If directory creation fails. 

231 """ 

232 path = validate_child_path(path, self.home_path) 

233 try: 

234 return run_ssh(self.server, 'mkdir', '-p', path) 

235 except subprocess.CalledProcessError as e: 

236 if e.returncode != 1: # assume it exists if exitcode is 1 

237 new = RemoteMkDirProcessError(e.returncode, e.cmd, output=e.output) 

238 raise new from e 

239 

240 def remote_pull(self): 

241 """Cascade pull to the server.""" 

242 return run_ssh('-A', self.server, 'hods', '--pull', capture_output=False) 

243 

244 def remote_push(self): 

245 """Cascade push to the server.""" 

246 return run_ssh('-A', self.server, 'hods', '--push', capture_output=False) 

247 

248 def _parse_pulled_at(self, data): 

249 if data is None: 

250 self.pulled_at = None 

251 return 

252 self.pulled_at = datetime.datetime.strptime(data, DATETIME_FORMAT) 

253 

254 def _serialize_pulled_at(self): 

255 if self.pulled_at is None: 

256 return 

257 return self.pulled_at.strftime(DATETIME_FORMAT) 

258 

259 

260class Config(BaseConfigFile): 

261 """Global configuration from the server.""" 

262 

263 def __init__(self, settings, load=True): 

264 """Initialize configuration.""" 

265 from hods.config.tree import SourceTree 

266 

267 super().__init__(os.path.join(settings.hods_path, 'config.json')) 

268 self.settings = settings 

269 self.tree = SourceTree(settings) 

270 

271 # variables are added/removed immediately in `pull` for consistency! 

272 # Use these temporary dictionaries to provide additional actions on them. 

273 self.new_variables = {} 

274 self.removed_variables = {} 

275 

276 if load: 

277 self.load() 

278 

279 @property 

280 def remote_path(self): 

281 """The path to the remote file prefixed with the server.""" 

282 return self.settings.get_server_path(self.path) 

283 

284 def pull(self): 

285 """Download and load the config from the given remote address.""" 

286 logger.info('pulling config from "%s"', self.settings.server) 

287 

288 p = run_rsync(self.remote_path, self.path) 

289 

290 self.load() 

291 self.settings.last_pull_at = datetime.datetime.now() 

292 self.clean_variables() 

293 self.save() 

294 return p 

295 

296 def push(self): 

297 """Save and upload the current config to the given remote address.""" 

298 logger.info('pushing config to "%s"', self.settings.server) 

299 self.settings.rmkdir('.hods') 

300 return run_rsync(self.path, self.remote_path) 

301 

302 def get_variables(self): 

303 """Collect all variables and their final values.""" 

304 for key, default in self.variables.items(): 

305 value = self.settings.variables.get(key, default) 

306 yield key, value or default 

307 

308 def clean_variables(self): 

309 """Clear removed variables and add new ones.""" 

310 removed = {} 

311 for key in list(self.settings.variables.keys()): 

312 if key not in self.variables: 

313 removed[key] = self.settings.variables[key] 

314 del self.settings.variables[key] 

315 self.removed_variables = removed 

316 

317 new = {} 

318 for key, default in self.variables.items(): 

319 if key not in self.settings.variables: 

320 new[key] = self.settings.variables[key] = default 

321 self.new_variables = new 

322 

323 @property 

324 def new_features(self): 

325 """Collect features that are not in the last configuration.""" 

326 for feature in self.tree.features: 

327 if self.settings.tree.get_child(feature.basename) is None: 

328 yield feature 

329 

330 def _parse_tree(self, data): 

331 for feature in data.get('features', ()): 

332 feature['installed'] = feature['name'] in self.settings.installed_features 

333 super()._parse_tree(data) 

334 

335 def _serialize_tree(self): 

336 data = super()._serialize_tree() 

337 installed_features = [] 

338 for feature in data['features']: 

339 if feature['installed']: 

340 installed_features.append(feature['name']) 

341 del feature['installed'] 

342 self.settings.installed_features = installed_features 

343 return data