3 # vmtools.py - part of the FDroid server tools
4 # Copyright (C) 2017 Michael Poehn <michael.poehn@fsfe.org>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from os import remove as rmfile
20 from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser
29 from .common import FDroidException
30 from logging import getLogger
32 logger = getLogger('fdroidserver-vmtools')
35 def get_clean_builder(serverdir, reset=False):
36 if not os.path.isdir(serverdir):
37 if os.path.islink(serverdir):
39 logger.info("buildserver path does not exists, creating %s", serverdir)
40 os.makedirs(serverdir)
41 vagrantfile = os.path.join(serverdir, 'Vagrantfile')
42 if not os.path.isfile(vagrantfile):
43 with open(os.path.join('builder', 'Vagrantfile'), 'w') as f:
44 f.write(textwrap.dedent("""\
45 # generated file, do not change.
47 Vagrant.configure("2") do |config|
48 config.vm.box = "buildserver"
49 config.vm.synced_folder ".", "/vagrant", disabled: true
52 vm = get_build_vm(serverdir)
54 logger.info('resetting buildserver by request')
55 elif not vm.vagrant_uuid_okay():
56 logger.info('resetting buildserver, bceause vagrant vm is not okay.')
58 elif not vm.snapshot_exists('fdroidclean'):
59 logger.info("resetting buildserver, because snapshot 'fdroidclean' is not present.")
68 logger.info('buildserver recreated: taking a clean snapshot')
69 vm.snapshot_create('fdroidclean')
71 logger.info('builserver ok: reverting to clean snapshot')
72 vm.snapshot_revert('fdroidclean')
76 sshinfo = vm.sshinfo()
77 except FDroidBuildVmException:
78 # workaround because libvirt sometimes likes to forget
79 # about ssh connection info even thou the vm is running
82 sshinfo = vm.sshinfo()
87 def _check_call(cmd, shell=False, cwd=None):
88 logger.debug(' '.join(cmd))
89 return subprocess.check_call(cmd, shell=shell, cwd=cwd)
92 def _check_output(cmd, shell=False, cwd=None):
93 logger.debug(' '.join(cmd))
94 return subprocess.check_output(cmd, shell=shell, cwd=cwd)
97 def get_build_vm(srvdir, provider=None):
98 """Factory function for getting FDroidBuildVm instances.
100 This function tries to figure out what hypervisor should be used
101 and creates an object for controlling a build VM.
103 :param srvdir: path to a directory which contains a Vagrantfile
104 :param provider: optionally this parameter allows specifiying an
105 spesific vagrant provider.
106 :returns: FDroidBuildVm instance.
108 abssrvdir = abspath(srvdir)
110 # use supplied provider
112 if provider == 'libvirt':
113 logger.debug('build vm provider \'libvirt\' selected')
114 return LibvirtBuildVm(abssrvdir)
115 elif provider == 'virtualbox':
116 logger.debug('build vm provider \'virtualbox\' selected')
117 return VirtualboxBuildVm(abssrvdir)
119 logger.warn('build vm provider not supported: \'%s\'', provider)
121 # try guessing provider from installed software
123 kvm_installed = 0 == _check_call(['which', 'kvm'])
124 except subprocess.CalledProcessError:
125 kvm_installed = False
127 kvm_installed |= 0 == _check_call(['which', 'qemu'])
128 except subprocess.CalledProcessError:
131 vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'])
132 except subprocess.CalledProcessError:
133 vbox_installed = False
134 if kvm_installed and vbox_installed:
135 logger.debug('both kvm and vbox are installed.')
137 logger.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'')
138 return LibvirtBuildVm(abssrvdir)
140 logger.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'')
141 return VirtualboxBuildVm(abssrvdir)
143 logger.debug('could not confirm that either virtualbox or kvm/libvirt are installed')
145 # try guessing provider from .../srvdir/.vagrant internals
146 has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant',
147 'machines', 'default', 'libvirt'))
148 has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant',
149 'machines', 'default', 'virtualbox'))
150 if has_libvirt_machine and has_vbox_machine:
151 logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
152 return VirtualboxBuildVm(abssrvdir)
153 elif has_libvirt_machine:
154 logger.debug('build vm provider lookup found \'libvirt\'')
155 return LibvirtBuildVm(abssrvdir)
156 elif has_vbox_machine:
157 logger.debug('build vm provider lookup found \'virtualbox\'')
158 return VirtualboxBuildVm(abssrvdir)
160 logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'')
161 return VirtualboxBuildVm(abssrvdir)
164 class FDroidBuildVmException(FDroidException):
168 class FDroidBuildVm():
169 """Abstract base class for working with FDroids build-servers.
171 Use the factory method `fdroidserver.vmtools.get_build_vm()` for
172 getting correct instances of this class.
174 This is intended to be a hypervisor independant, fault tolerant
175 wrapper around the vagrant functions we use.
178 def __init__(self, srvdir):
179 """Create new server class.
182 self.srvname = basename(srvdir) + '_default'
183 self.vgrntfile = joinpath(srvdir, 'Vagrantfile')
184 self.srvuuid = self._vagrant_fetch_uuid()
185 if not isdir(srvdir):
186 raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
187 if not isfile(self.vgrntfile):
188 raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
190 self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
192 def up(self, provision=True):
194 self.vgrnt.up(provision=provision)
195 logger.info('...waiting a sec...')
197 self.srvuuid = self._vagrant_fetch_uuid()
198 except subprocess.CalledProcessError as e:
199 raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
202 logger.info('suspending buildserver')
205 logger.info('...waiting a sec...')
207 except subprocess.CalledProcessError as e:
208 raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
211 self.vgrnt.halt(force=True)
214 """Remove every trace of this VM from the system.
216 This includes deleting:
217 * hypervisor specific definitions
218 * vagrant state informations (eg. `.vagrant` folder)
219 * images related to this vm
221 logger.info("destroying vm '%s'", self.srvname)
224 logger.debug('vagrant destroy completed')
225 except subprocess.CalledProcessError as e:
226 logger.exception('vagrant destroy failed: %s', e)
227 vgrntdir = joinpath(self.srvdir, '.vagrant')
229 shutil.rmtree(vgrntdir)
230 logger.debug('deleted vagrant dir: %s', vgrntdir)
231 except Exception as e:
232 logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e)
234 _check_call(['vagrant', 'global-status', '--prune'])
235 except subprocess.CalledProcessError as e:
236 logger.debug('pruning global vagrant status failed: %s', e)
238 def package(self, output=None):
239 self.vgrnt.package(output=output)
241 def vagrant_uuid_okay(self):
242 '''Having an uuid means that vagrant up has run successfully.'''
243 if self.srvuuid is None:
247 def _vagrant_file_name(self, name):
248 return name.replace('/', '-VAGRANTSLASH-')
250 def _vagrant_fetch_uuid(self):
251 if isfile(joinpath(self.srvdir, '.vagrant')):
252 # Vagrant 1.0 - it's a json file...
253 with open(joinpath(self.srvdir, '.vagrant')) as f:
254 id = json.load(f)['active']['default']
255 logger.debug('vm uuid: %s', id)
257 elif isfile(joinpath(self.srvdir, '.vagrant', 'machines',
258 'default', self.provider, 'id')):
259 # Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
260 with open(joinpath(self.srvdir, '.vagrant', 'machines',
261 'default', self.provider, 'id')) as f:
263 logger.debug('vm uuid: %s', id)
266 logger.debug('vm uuid is None')
269 def box_add(self, boxname, boxfile, force=True):
270 """Add vagrant box to vagrant.
272 :param boxname: name assigned to local deployment of box
273 :param boxfile: path to box file
274 :param force: overwrite existing box image (default: True)
276 boxfile = abspath(boxfile)
277 if not isfile(boxfile):
278 raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile)
279 self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
281 def box_remove(self, boxname):
283 _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
284 except subprocess.CalledProcessError as e:
285 logger.debug('tried removing box %s, but is did not exist: %s', boxname, e)
286 boxpath = joinpath(expanduser('~'), '.vagrant',
287 self._vagrant_file_name(boxname))
289 logger.info("attempting to remove box '%s' by deleting: %s",
291 shutil.rmtree(boxpath)
294 """Get ssh connection info for a vagrant VM
296 :returns: A dictionary containing 'hostname', 'port', 'user'
301 _check_call(['vagrant ssh-config > sshconfig'],
302 cwd=self.srvdir, shell=True)
303 vagranthost = 'default' # Host in ssh config file
304 sshconfig = paramiko.SSHConfig()
305 with open(joinpath(self.srvdir, 'sshconfig'), 'r') as f:
307 sshconfig = sshconfig.lookup(vagranthost)
308 idfile = sshconfig['identityfile']
309 if isinstance(idfile, list):
311 elif idfile.startswith('"') and idfile.endswith('"'):
312 idfile = idfile[1:-1]
313 return {'hostname': sshconfig['hostname'],
314 'port': int(sshconfig['port']),
315 'user': sshconfig['user'],
317 except subprocess.CalledProcessError as e:
318 raise FDroidBuildVmException("Error getting ssh config") from e
320 def snapshot_create(self, snapshot_name):
321 raise NotImplementedError('not implemented, please use a sub-type instance')
323 def snapshot_list(self):
324 raise NotImplementedError('not implemented, please use a sub-type instance')
326 def snapshot_exists(self, snapshot_name):
327 raise NotImplementedError('not implemented, please use a sub-type instance')
329 def snapshot_revert(self, snapshot_name):
330 raise NotImplementedError('not implemented, please use a sub-type instance')
333 class LibvirtBuildVm(FDroidBuildVm):
334 def __init__(self, srvdir):
335 self.provider = 'libvirt'
336 super().__init__(srvdir)
340 self.conn = libvirt.open('qemu:///system')
341 except libvirt.libvirtError as e:
342 raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e))
348 # resorting to virsh instead of libvirt python bindings, because
349 # this is way more easy and therefore fault tolerant.
350 # (eg. lookupByName only works on running VMs)
352 _check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname))
353 logger.info("...waiting a sec...")
355 except subprocess.CalledProcessError as e:
356 logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e)
358 # libvirt python bindings do not support all flags required
359 # for undefining domains correctly.
360 _check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
361 logger.info("...waiting a sec...")
363 except subprocess.CalledProcessError as e:
364 logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e)
366 def package(self, output=None, keep_box_file=False):
368 output = "buildserver.box"
369 logger.debug('no output name set for packaging \'%s\',' +
370 'defaulting to %s', self.srvname, output)
371 storagePool = self.conn.storagePoolLookupByName('default')
372 domainInfo = self.conn.lookupByName(self.srvname).info()
375 if isfile('metadata.json'):
376 rmfile('metadata.json')
377 if isfile('Vagrantfile'):
378 rmfile('Vagrantfile')
379 if isfile('box.img'):
382 logger.debug('preparing box.img for box %s', output)
383 vol = storagePool.storageVolLookupByName(self.srvname + '.img')
384 imagepath = vol.path()
385 # TODO use a libvirt storage pool to ensure the img file is readable
386 _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
387 shutil.copy2(imagepath, 'box.img')
388 _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
389 img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
390 img_info = json.loads(img_info_raw.decode('utf-8'))
391 metadata = {"provider": "libvirt",
392 "format": img_info['format'],
393 "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
396 logger.debug('preparing metadata.json for box %s', output)
397 with open('metadata.json', 'w') as fp:
398 fp.write(json.dumps(metadata))
399 logger.debug('preparing Vagrantfile for box %s', output)
400 vagrantfile = textwrap.dedent("""\
401 Vagrant.configure("2") do |config|
402 config.ssh.username = "vagrant"
403 config.ssh.password = "vagrant"
405 config.vm.provider :libvirt do |libvirt|
407 libvirt.driver = "kvm"
409 libvirt.connect_via_ssh = false
410 libvirt.storage_pool_name = "default"
411 libvirt.cpus = {cpus}
412 libvirt.memory = {memory}
415 end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])}))
416 with open('Vagrantfile', 'w') as fp:
417 fp.write(vagrantfile)
418 with tarfile.open(output, 'w:gz') as tar:
419 logger.debug('adding metadata.json to box %s ...', output)
420 tar.add('metadata.json')
421 logger.debug('adding Vagrantfile to box %s ...', output)
422 tar.add('Vagrantfile')
423 logger.debug('adding box.img to box %s ...', output)
426 if not keep_box_file:
427 logger.debug('box packaging complete, removing temporary files.')
428 rmfile('metadata.json')
429 rmfile('Vagrantfile')
433 logger.warn('could not connect to storage-pool \'default\',' +
434 'skipping packaging buildserver box')
436 def box_add(self, boxname, boxfile, force=True):
437 boximg = '%s_vagrant_box_image_0.img' % (boxname)
440 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', boximg])
441 logger.debug("removed old box image '%s' from libvirt storeage pool", boximg)
442 except subprocess.CalledProcessError as e:
443 logger.debug("tired removing old box image '%s', file was not present in first place", boximg, exc_info=e)
444 super().box_add(boxname, boxfile, force)
446 def box_remove(self, boxname):
447 super().box_remove(boxname)
449 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)])
450 except subprocess.CalledProcessError as e:
451 logger.debug("tired removing '%s', file was not present in first place", boxname, exc_info=e)
453 def snapshot_create(self, snapshot_name):
454 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
456 _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name])
457 logger.info('...waiting a sec...')
459 except subprocess.CalledProcessError as e:
460 raise FDroidBuildVmException("could not cerate snapshot '%s' "
462 % (snapshot_name, self.srvname)) from e
464 def snapshot_list(self):
467 dom = self.conn.lookupByName(self.srvname)
468 return dom.listAllSnapshots()
469 except libvirt.libvirtError as e:
470 raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e
472 def snapshot_exists(self, snapshot_name):
475 dom = self.conn.lookupByName(self.srvname)
476 return dom.snapshotLookupByName(snapshot_name) is not None
477 except libvirt.libvirtError:
480 def snapshot_revert(self, snapshot_name):
481 logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
484 dom = self.conn.lookupByName(self.srvname)
485 snap = dom.snapshotLookupByName(snapshot_name)
486 dom.revertToSnapshot(snap)
487 logger.info('...waiting a sec...')
489 except libvirt.libvirtError as e:
490 raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\''
491 % (self.srvname, snapshot_name)) from e
494 class VirtualboxBuildVm(FDroidBuildVm):
496 def __init__(self, srvdir):
497 self.provider = 'virtualbox'
498 super().__init__(srvdir)
500 def snapshot_create(self, snapshot_name):
501 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
503 _check_call(['VBoxManage', 'snapshot', self.srvuuid, 'take', 'fdroidclean'], cwd=self.srvdir)
504 logger.info('...waiting a sec...')
506 except subprocess.CalledProcessError as e:
507 raise FDroidBuildVmException('could not cerate snapshot '
508 'of virtualbox vm %s'
509 % self.srvname) from e
511 def snapshot_list(self):
513 o = _check_output(['VBoxManage', 'snapshot',
514 self.srvuuid, 'list',
515 '--details'], cwd=self.srvdir)
517 except subprocess.CalledProcessError as e:
518 raise FDroidBuildVmException("could not list snapshots "
519 "of virtualbox vm '%s'"
520 % (self.srvname)) from e
522 def snapshot_exists(self, snapshot_name):
524 return str(snapshot_name) in str(self.snapshot_list())
525 except FDroidBuildVmException:
528 def snapshot_revert(self, snapshot_name):
529 logger.info("reverting vm '%s' to snapshot '%s'",
530 self.srvname, snapshot_name)
532 _check_call(['VBoxManage', 'snapshot', self.srvuuid,
533 'restore', 'fdroidclean'], cwd=self.srvdir)
534 except subprocess.CalledProcessError as e:
535 raise FDroidBuildVmException("could not load snapshot "
536 "'fdroidclean' for vm '%s'"
537 % (self.srvname)) from e