3 # vmtools.py - part of the FDroid server tools
4 # Copyright (C) 2017 Michael Poehn <michael.poehn@fsfe.org>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from os import remove as rmfile
20 from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser
29 from .common import FDroidException
30 from logging import getLogger
32 from fdroidserver import _
34 logger = getLogger('fdroidserver-vmtools')
37 def get_clean_builder(serverdir, reset=False):
38 if not os.path.isdir(serverdir):
39 if os.path.islink(serverdir):
41 logger.info("buildserver path does not exists, creating %s", serverdir)
42 os.makedirs(serverdir)
43 vagrantfile = os.path.join(serverdir, 'Vagrantfile')
44 if not os.path.isfile(vagrantfile):
45 with open(os.path.join('builder', 'Vagrantfile'), 'w') as f:
46 f.write(textwrap.dedent("""\
47 # generated file, do not change.
49 Vagrant.configure("2") do |config|
50 config.vm.box = "buildserver"
51 config.vm.synced_folder ".", "/vagrant", disabled: true
54 vm = get_build_vm(serverdir)
56 logger.info('resetting buildserver by request')
57 elif not vm.vagrant_uuid_okay():
58 logger.info('resetting buildserver, bceause vagrant vm is not okay.')
60 elif not vm.snapshot_exists('fdroidclean'):
61 logger.info("resetting buildserver, because snapshot 'fdroidclean' is not present.")
70 logger.info('buildserver recreated: taking a clean snapshot')
71 vm.snapshot_create('fdroidclean')
73 logger.info('builserver ok: reverting to clean snapshot')
74 vm.snapshot_revert('fdroidclean')
78 sshinfo = vm.sshinfo()
79 except FDroidBuildVmException:
80 # workaround because libvirt sometimes likes to forget
81 # about ssh connection info even thou the vm is running
84 sshinfo = vm.sshinfo()
89 def _check_call(cmd, shell=False, cwd=None):
90 logger.debug(' '.join(cmd))
91 return subprocess.check_call(cmd, shell=shell, cwd=cwd)
94 def _check_output(cmd, shell=False, cwd=None):
95 logger.debug(' '.join(cmd))
96 return subprocess.check_output(cmd, shell=shell, cwd=cwd)
99 def get_build_vm(srvdir, provider=None):
100 """Factory function for getting FDroidBuildVm instances.
102 This function tries to figure out what hypervisor should be used
103 and creates an object for controlling a build VM.
105 :param srvdir: path to a directory which contains a Vagrantfile
106 :param provider: optionally this parameter allows specifiying an
107 spesific vagrant provider.
108 :returns: FDroidBuildVm instance.
110 abssrvdir = abspath(srvdir)
112 # use supplied provider
114 if provider == 'libvirt':
115 logger.debug('build vm provider \'libvirt\' selected')
116 return LibvirtBuildVm(abssrvdir)
117 elif provider == 'virtualbox':
118 logger.debug('build vm provider \'virtualbox\' selected')
119 return VirtualboxBuildVm(abssrvdir)
121 logger.warn('build vm provider not supported: \'%s\'', provider)
123 # try guessing provider from installed software
125 kvm_installed = 0 == _check_call(['which', 'kvm'])
126 except subprocess.CalledProcessError:
127 kvm_installed = False
129 kvm_installed |= 0 == _check_call(['which', 'qemu'])
130 except subprocess.CalledProcessError:
133 vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'])
134 except subprocess.CalledProcessError:
135 vbox_installed = False
136 if kvm_installed and vbox_installed:
137 logger.debug('both kvm and vbox are installed.')
139 logger.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'')
140 return LibvirtBuildVm(abssrvdir)
142 logger.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'')
143 return VirtualboxBuildVm(abssrvdir)
145 logger.debug('could not confirm that either virtualbox or kvm/libvirt are installed')
147 # try guessing provider from .../srvdir/.vagrant internals
148 has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant',
149 'machines', 'default', 'libvirt'))
150 has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant',
151 'machines', 'default', 'virtualbox'))
152 if has_libvirt_machine and has_vbox_machine:
153 logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
154 return VirtualboxBuildVm(abssrvdir)
155 elif has_libvirt_machine:
156 logger.debug('build vm provider lookup found \'libvirt\'')
157 return LibvirtBuildVm(abssrvdir)
158 elif has_vbox_machine:
159 logger.debug('build vm provider lookup found \'virtualbox\'')
160 return VirtualboxBuildVm(abssrvdir)
162 logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'')
163 return VirtualboxBuildVm(abssrvdir)
166 class FDroidBuildVmException(FDroidException):
170 class FDroidBuildVm():
171 """Abstract base class for working with FDroids build-servers.
173 Use the factory method `fdroidserver.vmtools.get_build_vm()` for
174 getting correct instances of this class.
176 This is intended to be a hypervisor independant, fault tolerant
177 wrapper around the vagrant functions we use.
180 def __init__(self, srvdir):
181 """Create new server class.
184 self.srvname = basename(srvdir) + '_default'
185 self.vgrntfile = joinpath(srvdir, 'Vagrantfile')
186 self.srvuuid = self._vagrant_fetch_uuid()
187 if not isdir(srvdir):
188 raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
189 if not isfile(self.vgrntfile):
190 raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
192 self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
194 def up(self, provision=True):
196 self.vgrnt.up(provision=provision)
197 logger.info('...waiting a sec...')
199 self.srvuuid = self._vagrant_fetch_uuid()
200 except subprocess.CalledProcessError as e:
201 raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
204 logger.info('suspending buildserver')
207 logger.info('...waiting a sec...')
209 except subprocess.CalledProcessError as e:
210 raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
213 self.vgrnt.halt(force=True)
216 """Remove every trace of this VM from the system.
218 This includes deleting:
219 * hypervisor specific definitions
220 * vagrant state informations (eg. `.vagrant` folder)
221 * images related to this vm
223 logger.info("destroying vm '%s'", self.srvname)
226 logger.debug('vagrant destroy completed')
227 except subprocess.CalledProcessError as e:
228 logger.exception('vagrant destroy failed: %s', e)
229 vgrntdir = joinpath(self.srvdir, '.vagrant')
231 shutil.rmtree(vgrntdir)
232 logger.debug('deleted vagrant dir: %s', vgrntdir)
233 except Exception as e:
234 logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e)
236 _check_call(['vagrant', 'global-status', '--prune'])
237 except subprocess.CalledProcessError as e:
238 logger.debug('pruning global vagrant status failed: %s', e)
240 def package(self, output=None):
241 self.vgrnt.package(output=output)
243 def vagrant_uuid_okay(self):
244 '''Having an uuid means that vagrant up has run successfully.'''
245 if self.srvuuid is None:
249 def _vagrant_file_name(self, name):
250 return name.replace('/', '-VAGRANTSLASH-')
252 def _vagrant_fetch_uuid(self):
253 if isfile(joinpath(self.srvdir, '.vagrant')):
254 # Vagrant 1.0 - it's a json file...
255 with open(joinpath(self.srvdir, '.vagrant')) as f:
256 id = json.load(f)['active']['default']
257 logger.debug('vm uuid: %s', id)
259 elif isfile(joinpath(self.srvdir, '.vagrant', 'machines',
260 'default', self.provider, 'id')):
261 # Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
262 with open(joinpath(self.srvdir, '.vagrant', 'machines',
263 'default', self.provider, 'id')) as f:
265 logger.debug('vm uuid: %s', id)
268 logger.debug('vm uuid is None')
271 def box_add(self, boxname, boxfile, force=True):
272 """Add vagrant box to vagrant.
274 :param boxname: name assigned to local deployment of box
275 :param boxfile: path to box file
276 :param force: overwrite existing box image (default: True)
278 boxfile = abspath(boxfile)
279 if not isfile(boxfile):
280 raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile)
281 self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
283 def box_remove(self, boxname):
285 _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
286 except subprocess.CalledProcessError as e:
287 logger.debug('tried removing box %s, but is did not exist: %s', boxname, e)
288 boxpath = joinpath(expanduser('~'), '.vagrant',
289 self._vagrant_file_name(boxname))
291 logger.info("attempting to remove box '%s' by deleting: %s",
293 shutil.rmtree(boxpath)
296 """Get ssh connection info for a vagrant VM
298 :returns: A dictionary containing 'hostname', 'port', 'user'
303 _check_call(['vagrant ssh-config > sshconfig'],
304 cwd=self.srvdir, shell=True)
305 vagranthost = 'default' # Host in ssh config file
306 sshconfig = paramiko.SSHConfig()
307 with open(joinpath(self.srvdir, 'sshconfig'), 'r') as f:
309 sshconfig = sshconfig.lookup(vagranthost)
310 idfile = sshconfig['identityfile']
311 if isinstance(idfile, list):
313 elif idfile.startswith('"') and idfile.endswith('"'):
314 idfile = idfile[1:-1]
315 return {'hostname': sshconfig['hostname'],
316 'port': int(sshconfig['port']),
317 'user': sshconfig['user'],
319 except subprocess.CalledProcessError as e:
320 raise FDroidBuildVmException("Error getting ssh config") from e
322 def snapshot_create(self, snapshot_name):
323 raise NotImplementedError('not implemented, please use a sub-type instance')
325 def snapshot_list(self):
326 raise NotImplementedError('not implemented, please use a sub-type instance')
328 def snapshot_exists(self, snapshot_name):
329 raise NotImplementedError('not implemented, please use a sub-type instance')
331 def snapshot_revert(self, snapshot_name):
332 raise NotImplementedError('not implemented, please use a sub-type instance')
335 class LibvirtBuildVm(FDroidBuildVm):
336 def __init__(self, srvdir):
337 self.provider = 'libvirt'
338 super().__init__(srvdir)
342 self.conn = libvirt.open('qemu:///system')
343 except libvirt.libvirtError as e:
344 raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e))
350 # resorting to virsh instead of libvirt python bindings, because
351 # this is way more easy and therefore fault tolerant.
352 # (eg. lookupByName only works on running VMs)
354 _check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname))
355 logger.info("...waiting a sec...")
357 except subprocess.CalledProcessError as e:
358 logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e)
360 # libvirt python bindings do not support all flags required
361 # for undefining domains correctly.
362 _check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
363 logger.info("...waiting a sec...")
365 except subprocess.CalledProcessError as e:
366 logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e)
368 def package(self, output=None, keep_box_file=False):
370 output = "buildserver.box"
371 logger.debug('no output name set for packaging \'%s\',' +
372 'defaulting to %s', self.srvname, output)
373 storagePool = self.conn.storagePoolLookupByName('default')
374 domainInfo = self.conn.lookupByName(self.srvname).info()
377 if isfile('metadata.json'):
378 rmfile('metadata.json')
379 if isfile('Vagrantfile'):
380 rmfile('Vagrantfile')
381 if isfile('box.img'):
384 logger.debug('preparing box.img for box %s', output)
385 vol = storagePool.storageVolLookupByName(self.srvname + '.img')
386 imagepath = vol.path()
387 # TODO use a libvirt storage pool to ensure the img file is readable
388 if not os.access(imagepath, os.R_OK):
389 logger.warning(_('Cannot read "{path}"!').format(path=imagepath))
390 _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
391 shutil.copy2(imagepath, 'box.img')
392 _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
393 img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
394 img_info = json.loads(img_info_raw.decode('utf-8'))
395 metadata = {"provider": "libvirt",
396 "format": img_info['format'],
397 "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
400 logger.debug('preparing metadata.json for box %s', output)
401 with open('metadata.json', 'w') as fp:
402 fp.write(json.dumps(metadata))
403 logger.debug('preparing Vagrantfile for box %s', output)
404 vagrantfile = textwrap.dedent("""\
405 Vagrant.configure("2") do |config|
406 config.ssh.username = "vagrant"
407 config.ssh.password = "vagrant"
409 config.vm.provider :libvirt do |libvirt|
411 libvirt.driver = "kvm"
413 libvirt.connect_via_ssh = false
414 libvirt.storage_pool_name = "default"
415 libvirt.cpus = {cpus}
416 libvirt.memory = {memory}
419 end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])}))
420 with open('Vagrantfile', 'w') as fp:
421 fp.write(vagrantfile)
422 with tarfile.open(output, 'w:gz') as tar:
423 logger.debug('adding metadata.json to box %s ...', output)
424 tar.add('metadata.json')
425 logger.debug('adding Vagrantfile to box %s ...', output)
426 tar.add('Vagrantfile')
427 logger.debug('adding box.img to box %s ...', output)
430 if not keep_box_file:
431 logger.debug('box packaging complete, removing temporary files.')
432 rmfile('metadata.json')
433 rmfile('Vagrantfile')
437 logger.warn('could not connect to storage-pool \'default\',' +
438 'skipping packaging buildserver box')
440 def box_add(self, boxname, boxfile, force=True):
441 boximg = '%s_vagrant_box_image_0.img' % (boxname)
444 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', boximg])
445 logger.debug("removed old box image '%s' from libvirt storeage pool", boximg)
446 except subprocess.CalledProcessError as e:
447 logger.debug("tired removing old box image '%s', file was not present in first place", boximg, exc_info=e)
448 super().box_add(boxname, boxfile, force)
450 def box_remove(self, boxname):
451 super().box_remove(boxname)
453 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)])
454 except subprocess.CalledProcessError as e:
455 logger.debug("tired removing '%s', file was not present in first place", boxname, exc_info=e)
457 def snapshot_create(self, snapshot_name):
458 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
460 _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name])
461 logger.info('...waiting a sec...')
463 except subprocess.CalledProcessError as e:
464 raise FDroidBuildVmException("could not cerate snapshot '%s' "
466 % (snapshot_name, self.srvname)) from e
468 def snapshot_list(self):
471 dom = self.conn.lookupByName(self.srvname)
472 return dom.listAllSnapshots()
473 except libvirt.libvirtError as e:
474 raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e
476 def snapshot_exists(self, snapshot_name):
479 dom = self.conn.lookupByName(self.srvname)
480 return dom.snapshotLookupByName(snapshot_name) is not None
481 except libvirt.libvirtError:
484 def snapshot_revert(self, snapshot_name):
485 logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
488 dom = self.conn.lookupByName(self.srvname)
489 snap = dom.snapshotLookupByName(snapshot_name)
490 dom.revertToSnapshot(snap)
491 logger.info('...waiting a sec...')
493 except libvirt.libvirtError as e:
494 raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\''
495 % (self.srvname, snapshot_name)) from e
498 class VirtualboxBuildVm(FDroidBuildVm):
500 def __init__(self, srvdir):
501 self.provider = 'virtualbox'
502 super().__init__(srvdir)
504 def snapshot_create(self, snapshot_name):
505 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
507 _check_call(['VBoxManage', 'snapshot', self.srvuuid, 'take', 'fdroidclean'], cwd=self.srvdir)
508 logger.info('...waiting a sec...')
510 except subprocess.CalledProcessError as e:
511 raise FDroidBuildVmException('could not cerate snapshot '
512 'of virtualbox vm %s'
513 % self.srvname) from e
515 def snapshot_list(self):
517 o = _check_output(['VBoxManage', 'snapshot',
518 self.srvuuid, 'list',
519 '--details'], cwd=self.srvdir)
521 except subprocess.CalledProcessError as e:
522 raise FDroidBuildVmException("could not list snapshots "
523 "of virtualbox vm '%s'"
524 % (self.srvname)) from e
526 def snapshot_exists(self, snapshot_name):
528 return str(snapshot_name) in str(self.snapshot_list())
529 except FDroidBuildVmException:
532 def snapshot_revert(self, snapshot_name):
533 logger.info("reverting vm '%s' to snapshot '%s'",
534 self.srvname, snapshot_name)
536 _check_call(['VBoxManage', 'snapshot', self.srvuuid,
537 'restore', 'fdroidclean'], cwd=self.srvdir)
538 except subprocess.CalledProcessError as e:
539 raise FDroidBuildVmException("could not load snapshot "
540 "'fdroidclean' for vm '%s'"
541 % (self.srvname)) from e