Coverage for src/hods/config/sources.py: 100.00%

1"""hods - home directory synchronization. 

2 

3Copyright (C) 2016-2020 Mathias Stelzer <knoppo@rolln.de> 

4 

5hods is free software: you can redistribute it and/or modify 

6it under the terms of the GNU General Public License as published by 

7the Free Software Foundation, either version 3 of the License, or 

8(at your option) any later version. 

9 

10hods is distributed in the hope that it will be useful, 

11but WITHOUT ANY WARRANTY; without even the implied warranty of 

12MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

13GNU General Public License for more details. 

14 

15You should have received a copy of the GNU General Public License 

16along with this program. If not, see <http://www.gnu.org/licenses/>. 

17""" 

18import datetime 

19import logging 

20import os 

21import subprocess 

22 

23from hods.config.sourcefile import BaseSourceDirectory 

24from hods.utils import GitError, Sortable, run_git, run_rsync 

25 

26logger = logging.getLogger(__name__) 

27 

28 

29SOURCE_MAP = {} 

30 

31 

32class UnknownSource(Exception): 

33 """Exception raised when requesting a non-existent source class. 

34 

35 All valid source names are stored in `SOURCE_MAP`. 

36 """ 

37 

38 pass 

39 

40 

41def get_source_class(source_type): 

42 """Get source class for the given name. 

43 

44 Raises: 

45 UnknownSource: if given source_type is not registered. 

46 """ 

47 try: 

48 return SOURCE_MAP[source_type] 

49 except KeyError: 

50 raise UnknownSource('No source class registered for given type: "{}"'.format(source_type)) 

51 

52 

53class AlreadyRegisteredSource(Exception): 

54 """Exception raised when registering a source type that already exists.""" 

55 

56 pass 

57 

58 

59class SourceError(Exception): 

60 """Base class for all source errors.""" 

61 

62 pass 

63 

64 

65class Source(Sortable, BaseSourceDirectory): 

66 """The root source directory, synchronized with the server using rsync.""" 

67 

68 is_source_root = True 

69 

70 source_type = None 

71 

72 dependency = None 

73 

74 stored_on_master = True 

75 

76 @classmethod 

77 def register(cls): 

78 """Register this source class.""" 

79 if cls.source_type in SOURCE_MAP: 

80 raise AlreadyRegisteredSource('Cannot register {}. source_type "{}" is already registered.'.format( 

81 cls.__name__, cls.source_type)) 

82 SOURCE_MAP[cls.source_type] = cls 

83 

84 @classmethod 

85 def is_dependency_installed(cls): 

86 """Check if the required cmdline tool is installed.""" 

87 if cls.dependency is None: 

88 return True 

89 

90 from hods.utils import which 

91 return which(cls.dependency) is not None 

92 

93 def __init__(self, feature, name, **kwargs): 

94 """Initialize source. 

95 

96 :param feature: `hods.app.Feature` - Parent feature instance 

97 :param name: `str` - file basename 

98 :param kwargs: Additional options, passed to `hods.file.SourceFile` 

99 """ 

100 self.feature = feature 

101 self.tree = feature.tree 

102 self.pull_only = kwargs.pop('pull_only', False) 

103 super().__init__(feature, name, **kwargs) 

104 

105 def is_ignored(self): 

106 """Check if this instance should be synchronized.""" 

107 if not self.feature.installed: 

108 return True 

109 return super().is_ignored() 

110 

111 def as_dict(self): 

112 """Return the source as a dictionary to store.""" 

113 data = super().as_dict() 

114 if self.source_type: 

115 data['type'] = self.source_type 

116 data['pull_only'] = self.pull_only 

117 return data 

118 

119 @property 

120 def relative_source_path(self): 

121 """Return the basename as the relative path.""" 

122 return '' 

123 

124 @property 

125 def relative_feature_path(self): 

126 """Return the relative path to the parent feature.""" 

127 return self.basename 

128 

129 @property 

130 def relative_config_path(self): 

131 """Return the relative path to the parent config tree.""" 

132 return os.path.join(self.feature.basename, self.basename) 

133 

134 @property 

135 def default_destination(self): 

136 """Return an empty string to use the home directory itself as default destination.""" 

137 return '' 

138 

139 @property 

140 def home_relpath(self): 

141 """The relative path to the home directory.""" 

142 return self.relpath(self.tree.settings.home_path) 

143 

144 @property 

145 def remote_path(self): 

146 """Return the host:path combination string.""" 

147 return self.tree.settings.get_server_path(self.home_relpath) 

148 

149 def _rsync(self, src, dst, **kwargs): 

150 """Run a git subcommand in this repository.""" 

151 kwargs.setdefault('cwd', self.feature.path) 

152 return run_rsync(src, dst, **kwargs) 

153 

154 def pull(self): 

155 """Run rsync to download this directory from the server. 

156 

157 Returns: The completed process information 

158 

159 Raises: 

160 subprocess.CalledProcessError: If the command returns a non-zero exitcode. 

161 """ 

162 if not self.feature.exists(): 

163 self.feature.mkdir() 

164 process = self._rsync(self.remote_path + os.sep, self.basename + os.sep) 

165 self.scan(clear=True, recursive=True) 

166 return process 

167 

168 def push(self): 

169 """Run rsync to upload this directory to the server. 

170 

171 Returns: The completed process information 

172 

173 Raises: 

174 subprocess.CalledProcessError: If the command returns a non-zero exitcode. 

175 """ 

176 self.tree.settings.rmkdir(self.home_relpath) 

177 return self._rsync(self.basename + os.sep, self.remote_path + os.sep) 

178 

179 def init(self): 

180 """Create the directory.""" 

181 if not self.feature.exists(): 

182 self.feature.mkdir() 

183 self.mkdir() 

184 

185 

186Source.register() 

187 

188 

189class NoChanges(SourceError): 

190 """Raised when nothing to push (no new commits).""" 

191 

192 pass 

193 

194 

195class AlreadyClonedError(GitError): 

196 """Raised when trying to clone an already existing repository.""" 

197 

198 pass 

199 

200 

201class GitPushError(GitError): 

202 """Raised when push of a git repository fails.""" 

203 pass 

204 

205 

206class GitRepository(Source): 

207 """A git repository as a source.""" 

208 

209 source_type = 'git' 

210 

211 dependency = 'git' 

212 

213 ignore_basenames = [ 

214 '.git', 

215 '.gitignore', 

216 ] 

217 

218 stored_on_master = False 

219 

220 @classmethod 

221 def basename_by_url(cls, url): 

222 """Parse the given ``url`` and extract the basename without suffix. 

223 

224 Args: 

225 url: git repository URL 

226 

227 Returns: basename for a local repository clone 

228 

229 Examples: 

230 >>> GitRepository.basename_by_url('git://user@example.com/dotfiles') 

231 'dotfiles' 

232 >>> GitRepository.basename_by_url('https://example.com/gitproject.git') 

233 'gitproject' 

234 >>> GitRepository.basename_by_url('user@example.com:project.git') 

235 'project' 

236 """ 

237 if ':' in url: # ssh-protocol with relative path ([user@]server:project.git) 

238 url = url.split(':')[-1] 

239 basename = url.split('/')[-1] # last url part 

240 if basename.endswith('.git'): # cut .git 

241 basename = basename[:-4] 

242 return basename 

243 

244 @classmethod 

245 def url_for_path(cls, path): 

246 """Return the origin remote url for the git repository at the given path. 

247 

248 Args: 

249 path: Path to the git repository 

250 

251 Returns: First stdout line of command: ``git config --get remote.origin.url`` 

252 

253 Raises: 

254 GitError: If git is not installed. 

255 subprocess.CalledProcessError: If the command returns a non-zero exitcode. 

256 """ 

257 if os.path.basename(path) == '.git': 

258 path = os.path.dirname(path) 

259 

260 p = run_git('config', '--get', 'remote.origin.url', cwd=path, hide=True) 

261 return p.stdout.splitlines()[0] 

262 

263 def __init__(self, feature, url, name=None, **kwargs): 

264 """Initialize git repository. 

265 

266 :param url: `str` - url to the repository 

267 """ 

268 if name is None: 

269 if url is None: 

270 raise TypeError('{} requires a name or an url argument.'.format(self.__class__.__name__)) 

271 name = self.basename_by_url(url) 

272 

273 super().__init__(feature, name, **kwargs) 

274 

275 # allow passing `Ǹone` only explicitly since it may raise `GitError` 

276 if url is None: 

277 url = self.url_for_path(self.path) 

278 self.url = url 

279 

280 def as_dict(self): 

281 """Return the source as a dictionary to store.""" 

282 data = super().as_dict() 

283 data['url'] = self.url 

284 return data 

285 

286 def _git(self, *cmd, **kwargs): 

287 """Run a git subcommand in this repository.""" 

288 kwargs.setdefault('cwd', self.path) 

289 return run_git(*cmd, **kwargs) 

290 

291 def list_unstaged(self, include_untracked=True): 

292 """List files not in index (unstaged).""" 

293 p = self._git('--no-pager', 'diff', '--name-only', hide=True) 

294 yield from p.stdout.splitlines() 

295 if include_untracked: 

296 yield from self.list_untracked() 

297 

298 def list_untracked(self): 

299 """List files unknown to git.""" 

300 p = self._git('--no-pager', 'ls-files', '--other', '--exclude-standard', hide=True) 

301 return p.stdout.splitlines() 

302 

303 def list_staged(self): 

304 """List files in index (staged).""" 

305 p = self._git('--no-pager', 'diff', '--name-only', '--cached', hide=True) 

306 return p.stdout.splitlines() 

307 

308 def add(self, *paths): 

309 """Add file contents to index. 

310 

311 If no files are given, use ``--all`` flag to add all changes. 

312 """ 

313 cmd = ('add',) 

314 if paths: 

315 cmd += paths 

316 else: 

317 cmd += ('--all',) 

318 return self._git(*cmd) 

319 

320 def commit(self, message=None, show_editor=True): 

321 """Commit current state. 

322 

323 :param message: `str` or callable - The commit message or callback to get it. 

324 :param show_editor: `bool` - Show the editor with the (default) commit message. 

325 :return: 

326 """ 

327 cmd = ['commit'] 

328 

329 if callable(message): 

330 message = message() 

331 

332 edit = message is None or show_editor 

333 if edit: 

334 cmd.append('--edit') 

335 

336 timestamp = datetime.datetime.utcnow().isoformat() 

337 

338 comment = 'hods commit UTC: ' + timestamp 

339 if show_editor: 

340 comment = '# ' + comment 

341 

342 if message is None: 

343 message = comment 

344 else: 

345 message = comment + '\n\n' + message 

346 

347 cmd.extend(['-m', message]) 

348 

349 return self._git(*cmd, capture_output=not edit) 

350 

351 def _get_local(self): 

352 try: 

353 return self._git('rev-parse', '@', hide=True).stdout 

354 except subprocess.CalledProcessError as e: 

355 if e.returncode == 128: 

356 raise NoChanges('No files in repository!') 

357 

358 def _get_remote(self): 

359 try: 

360 return self._git('rev-parse', '@{u}', hide=True).stdout 

361 except subprocess.CalledProcessError as e: 

362 if e.returncode != 128: 

363 raise e 

364 logger.info('origin is empty!') 

365 

366 def _get_base(self): 

367 try: 

368 return self._git('merge-base', '@', '@{u}', hide=True).stdout 

369 except subprocess.CalledProcessError as e: 

370 if e.returncode != 128: 

371 raise e 

372 

373 def validate_url(self): 

374 """Raise if no URL is set.""" 

375 if self.url and self.url.strip(): 

376 return 

377 raise GitError('git source has no URL assigned!') 

378 

379 def validate_upstream_push(self): 

380 """Check local and upstream states and whether a push is possible.""" 

381 local = self._get_local() 

382 if local is None: 

383 return False 

384 

385 remote = self._get_remote() 

386 if remote is None: 

387 return False 

388 

389 base = self._get_base() 

390 

391 if local == remote: 

392 raise NoChanges('No changes to push!') 

393 if local == base: 

394 raise GitPushError('New commits in upstream! Pull first.') 

395 if remote != base: 

396 raise GitPushError('Local and upstream have diverged! Please fix manually.') 

397 

398 return True 

399 

400 def is_cloned(self): 

401 """Return a `bool` whether this repository already exists.""" 

402 return os.path.exists(self.join('.git')) 

403 

404 def clone(self): 

405 """Clone the repository.""" 

406 if self.is_cloned(): 

407 raise AlreadyClonedError() 

408 

409 self.validate_url() 

410 

411 if not self.feature.exists(): 

412 self.feature.mkdir() 

413 

414 process = self._git('clone', self.url, cwd=self.feature.path) 

415 self.scan(clear=True) 

416 return process 

417 

418 def pull(self): 

419 """Pull the repository.""" 

420 if self.tree.settings.server_is_proxy: 

421 return super().pull() 

422 

423 if not self.is_cloned(): 

424 return self.clone() 

425 

426 self.validate_url() 

427 

428 process = self._git('pull') 

429 self.scan(clear=True) 

430 return process 

431 

432 def push(self, commit_callback=None, show_commit_editor=True): 

433 """Commit if necessary and push the git repository. 

434 

435 :return: ``None`` 

436 """ 

437 if self.tree.settings.server_is_proxy: 

438 return super().push() 

439 

440 if not self.is_cloned(): 

441 raise GitPushError('Not a git repository!') 

442 

443 self.validate_url() 

444 

445 unstaged_files = list(self.list_unstaged()) 

446 staged_files = self.list_staged() 

447 

448 if unstaged_files or staged_files: 

449 self.add() 

450 self.commit(commit_callback, show_commit_editor) 

451 

452 cmd = ['push'] 

453 

454 if not self.validate_upstream_push(): 

455 cmd.extend(['-u', 'origin', 'master']) 

456 

457 return self._git(*cmd) 

458 

459 def init(self): 

460 """Create the directory.""" 

461 super().init() 

462 if not self.tree.settings.server_is_proxy: 

463 self._git('init') 

464 self._git('remote', 'add', 'origin', self.url) 

465 

466 

467GitRepository.register()