3 # vmtools.py - part of the FDroid server tools
4 # Copyright (C) 2017 Michael Poehn <michael.poehn@fsfe.org>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from os.path import isdir, isfile, basename, abspath, expanduser
27 from .common import FDroidException
28 from logging import getLogger
30 from fdroidserver import _
33 lock = threading.Lock()
35 logger = getLogger('fdroidserver-vmtools')
38 def get_clean_builder(serverdir, reset=False):
39 if not os.path.isdir(serverdir):
40 if os.path.islink(serverdir):
42 logger.info("buildserver path does not exists, creating %s", serverdir)
43 os.makedirs(serverdir)
44 vagrantfile = os.path.join(serverdir, 'Vagrantfile')
45 if not os.path.isfile(vagrantfile):
46 with open(os.path.join('builder', 'Vagrantfile'), 'w') as f:
47 f.write(textwrap.dedent("""\
48 # generated file, do not change.
50 Vagrant.configure("2") do |config|
51 config.vm.box = "buildserver"
52 config.vm.synced_folder ".", "/vagrant", disabled: true
55 vm = get_build_vm(serverdir)
57 logger.info('resetting buildserver by request')
58 elif not vm.vagrant_uuid_okay():
59 logger.info('resetting buildserver, because vagrant vm is not okay.')
61 elif not vm.snapshot_exists('fdroidclean'):
62 logger.info("resetting buildserver, because snapshot 'fdroidclean' is not present.")
71 logger.info('buildserver recreated: taking a clean snapshot')
72 vm.snapshot_create('fdroidclean')
74 logger.info('builserver ok: reverting to clean snapshot')
75 vm.snapshot_revert('fdroidclean')
79 sshinfo = vm.sshinfo()
80 except FDroidBuildVmException:
81 # workaround because libvirt sometimes likes to forget
82 # about ssh connection info even thou the vm is running
85 sshinfo = vm.sshinfo()
90 def _check_call(cmd, cwd=None):
91 logger.debug(' '.join(cmd))
92 return subprocess.check_call(cmd, shell=False, cwd=cwd)
95 def _check_output(cmd, cwd=None):
96 logger.debug(' '.join(cmd))
97 return subprocess.check_output(cmd, shell=False, cwd=cwd)
100 def get_build_vm(srvdir, provider=None):
101 """Factory function for getting FDroidBuildVm instances.
103 This function tries to figure out what hypervisor should be used
104 and creates an object for controlling a build VM.
106 :param srvdir: path to a directory which contains a Vagrantfile
107 :param provider: optionally this parameter allows specifiying an
108 spesific vagrant provider.
109 :returns: FDroidBuildVm instance.
111 abssrvdir = abspath(srvdir)
113 # use supplied provider
115 if provider == 'libvirt':
116 logger.debug('build vm provider \'libvirt\' selected')
117 return LibvirtBuildVm(abssrvdir)
118 elif provider == 'virtualbox':
119 logger.debug('build vm provider \'virtualbox\' selected')
120 return VirtualboxBuildVm(abssrvdir)
122 logger.warn('build vm provider not supported: \'%s\'', provider)
124 # try guessing provider from installed software
126 kvm_installed = 0 == _check_call(['which', 'kvm'])
127 except subprocess.CalledProcessError:
128 kvm_installed = False
130 kvm_installed |= 0 == _check_call(['which', 'qemu'])
131 except subprocess.CalledProcessError:
134 vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'])
135 except subprocess.CalledProcessError:
136 vbox_installed = False
137 if kvm_installed and vbox_installed:
138 logger.debug('both kvm and vbox are installed.')
140 logger.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'')
141 return LibvirtBuildVm(abssrvdir)
143 logger.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'')
144 return VirtualboxBuildVm(abssrvdir)
146 logger.debug('could not confirm that either virtualbox or kvm/libvirt are installed')
148 # try guessing provider from .../srvdir/.vagrant internals
149 has_libvirt_machine = isdir(os.path.join(abssrvdir, '.vagrant',
150 'machines', 'default', 'libvirt'))
151 has_vbox_machine = isdir(os.path.join(abssrvdir, '.vagrant',
152 'machines', 'default', 'virtualbox'))
153 if has_libvirt_machine and has_vbox_machine:
154 logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
155 return VirtualboxBuildVm(abssrvdir)
156 elif has_libvirt_machine:
157 logger.debug('build vm provider lookup found \'libvirt\'')
158 return LibvirtBuildVm(abssrvdir)
159 elif has_vbox_machine:
160 logger.debug('build vm provider lookup found \'virtualbox\'')
161 return VirtualboxBuildVm(abssrvdir)
163 logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'')
164 return VirtualboxBuildVm(abssrvdir)
167 class FDroidBuildVmException(FDroidException):
171 class FDroidBuildVm():
172 """Abstract base class for working with FDroids build-servers.
174 Use the factory method `fdroidserver.vmtools.get_build_vm()` for
175 getting correct instances of this class.
177 This is intended to be a hypervisor independant, fault tolerant
178 wrapper around the vagrant functions we use.
180 def __init__(self, srvdir):
181 """Create new server class.
184 self.srvname = basename(srvdir) + '_default'
185 self.vgrntfile = os.path.join(srvdir, 'Vagrantfile')
186 self.srvuuid = self._vagrant_fetch_uuid()
187 if not isdir(srvdir):
188 raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
189 if not isfile(self.vgrntfile):
190 raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
192 self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
194 def up(self, provision=True):
198 self.vgrnt.up(provision=provision)
199 self.srvuuid = self._vagrant_fetch_uuid()
200 except subprocess.CalledProcessError as e:
201 raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
206 logger.info('suspending buildserver')
209 except subprocess.CalledProcessError as e:
210 raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
215 self.vgrnt.halt(force=True)
218 """Remove every trace of this VM from the system.
220 This includes deleting:
221 * hypervisor specific definitions
222 * vagrant state informations (eg. `.vagrant` folder)
223 * images related to this vm
225 logger.info("destroying vm '%s'", self.srvname)
228 logger.debug('vagrant destroy completed')
229 except subprocess.CalledProcessError as e:
230 logger.exception('vagrant destroy failed: %s', e)
231 vgrntdir = os.path.join(self.srvdir, '.vagrant')
233 shutil.rmtree(vgrntdir)
234 logger.debug('deleted vagrant dir: %s', vgrntdir)
235 except Exception as e:
236 logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e)
238 _check_call(['vagrant', 'global-status', '--prune'])
239 except subprocess.CalledProcessError as e:
240 logger.debug('pruning global vagrant status failed: %s', e)
242 def package(self, output=None):
243 self.vgrnt.package(output=output)
245 def vagrant_uuid_okay(self):
246 '''Having an uuid means that vagrant up has run successfully.'''
247 if self.srvuuid is None:
251 def _vagrant_file_name(self, name):
252 return name.replace('/', '-VAGRANTSLASH-')
254 def _vagrant_fetch_uuid(self):
255 if isfile(os.path.join(self.srvdir, '.vagrant')):
256 # Vagrant 1.0 - it's a json file...
257 with open(os.path.join(self.srvdir, '.vagrant')) as f:
258 id = json.load(f)['active']['default']
259 logger.debug('vm uuid: %s', id)
261 elif isfile(os.path.join(self.srvdir, '.vagrant', 'machines',
262 'default', self.provider, 'id')):
263 # Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
264 with open(os.path.join(self.srvdir, '.vagrant', 'machines',
265 'default', self.provider, 'id')) as f:
267 logger.debug('vm uuid: %s', id)
270 logger.debug('vm uuid is None')
273 def box_add(self, boxname, boxfile, force=True):
274 """Add vagrant box to vagrant.
276 :param boxname: name assigned to local deployment of box
277 :param boxfile: path to box file
278 :param force: overwrite existing box image (default: True)
280 boxfile = abspath(boxfile)
281 if not isfile(boxfile):
282 raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile)
283 self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
285 def box_remove(self, boxname):
287 _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
288 except subprocess.CalledProcessError as e:
289 logger.debug('tried removing box %s, but is did not exist: %s', boxname, e)
290 boxpath = os.path.join(expanduser('~'), '.vagrant',
291 self._vagrant_file_name(boxname))
293 logger.info("attempting to remove box '%s' by deleting: %s",
295 shutil.rmtree(boxpath)
298 """Get ssh connection info for a vagrant VM
300 :returns: A dictionary containing 'hostname', 'port', 'user'
305 sshconfig_path = os.path.join(self.srvdir, 'sshconfig')
306 with open(sshconfig_path, 'wb') as fp:
307 fp.write(_check_output(['vagrant', 'ssh-config'],
309 vagranthost = 'default' # Host in ssh config file
310 sshconfig = paramiko.SSHConfig()
311 with open(sshconfig_path, 'r') as f:
313 sshconfig = sshconfig.lookup(vagranthost)
314 idfile = sshconfig['identityfile']
315 if isinstance(idfile, list):
317 elif idfile.startswith('"') and idfile.endswith('"'):
318 idfile = idfile[1:-1]
319 return {'hostname': sshconfig['hostname'],
320 'port': int(sshconfig['port']),
321 'user': sshconfig['user'],
323 except subprocess.CalledProcessError as e:
324 raise FDroidBuildVmException("Error getting ssh config") from e
326 def snapshot_create(self, snapshot_name):
327 raise NotImplementedError('not implemented, please use a sub-type instance')
329 def snapshot_list(self):
330 raise NotImplementedError('not implemented, please use a sub-type instance')
332 def snapshot_exists(self, snapshot_name):
333 raise NotImplementedError('not implemented, please use a sub-type instance')
335 def snapshot_revert(self, snapshot_name):
336 raise NotImplementedError('not implemented, please use a sub-type instance')
339 class LibvirtBuildVm(FDroidBuildVm):
340 def __init__(self, srvdir):
341 self.provider = 'libvirt'
342 super().__init__(srvdir)
346 self.conn = libvirt.open('qemu:///system')
347 except libvirt.libvirtError as e:
348 raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e))
354 # resorting to virsh instead of libvirt python bindings, because
355 # this is way more easy and therefore fault tolerant.
356 # (eg. lookupByName only works on running VMs)
358 _check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname))
359 except subprocess.CalledProcessError as e:
360 logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e)
362 # libvirt python bindings do not support all flags required
363 # for undefining domains correctly.
364 _check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
365 except subprocess.CalledProcessError as e:
366 logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e)
368 def package(self, output=None, keep_box_file=False):
370 output = "buildserver.box"
371 logger.debug('no output name set for packaging \'%s\',' +
372 'defaulting to %s', self.srvname, output)
373 storagePool = self.conn.storagePoolLookupByName('default')
374 domainInfo = self.conn.lookupByName(self.srvname).info()
377 if isfile('metadata.json'):
378 os.remove('metadata.json')
379 if isfile('Vagrantfile'):
380 os.remove('Vagrantfile')
381 if isfile('box.img'):
384 logger.debug('preparing box.img for box %s', output)
385 vol = storagePool.storageVolLookupByName(self.srvname + '.img')
386 imagepath = vol.path()
387 # TODO use a libvirt storage pool to ensure the img file is readable
388 if not os.access(imagepath, os.R_OK):
389 logger.warning(_('Cannot read "{path}"!').format(path=imagepath))
390 _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
391 shutil.copy2(imagepath, 'box.img')
392 _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
393 img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
394 img_info = json.loads(img_info_raw.decode('utf-8'))
395 metadata = {"provider": "libvirt",
396 "format": img_info['format'],
397 "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
400 logger.debug('preparing metadata.json for box %s', output)
401 with open('metadata.json', 'w') as fp:
402 fp.write(json.dumps(metadata))
403 logger.debug('preparing Vagrantfile for box %s', output)
404 vagrantfile = textwrap.dedent("""\
405 Vagrant.configure("2") do |config|
406 config.ssh.username = "vagrant"
407 config.ssh.password = "vagrant"
409 config.vm.provider :libvirt do |libvirt|
411 libvirt.driver = "kvm"
413 libvirt.connect_via_ssh = false
414 libvirt.storage_pool_name = "default"
415 libvirt.cpus = {cpus}
416 libvirt.memory = {memory}
419 end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])}))
420 with open('Vagrantfile', 'w') as fp:
421 fp.write(vagrantfile)
422 with tarfile.open(output, 'w:gz') as tar:
423 logger.debug('adding metadata.json to box %s ...', output)
424 tar.add('metadata.json')
425 logger.debug('adding Vagrantfile to box %s ...', output)
426 tar.add('Vagrantfile')
427 logger.debug('adding box.img to box %s ...', output)
430 if not keep_box_file:
431 logger.debug('box packaging complete, removing temporary files.')
432 os.remove('metadata.json')
433 os.remove('Vagrantfile')
437 logger.warn('could not connect to storage-pool \'default\',' +
438 'skipping packaging buildserver box')
440 def box_add(self, boxname, boxfile, force=True):
441 boximg = '%s_vagrant_box_image_0.img' % (boxname)
444 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', boximg])
445 logger.debug("removed old box image '%s' from libvirt storeage pool", boximg)
446 except subprocess.CalledProcessError as e:
447 logger.debug("tired removing old box image '%s', file was not present in first place", boximg, exc_info=e)
448 super().box_add(boxname, boxfile, force)
450 def box_remove(self, boxname):
451 super().box_remove(boxname)
453 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)])
454 except subprocess.CalledProcessError as e:
455 logger.debug("tired removing '%s', file was not present in first place", boxname, exc_info=e)
457 def snapshot_create(self, snapshot_name):
458 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
460 _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name])
461 except subprocess.CalledProcessError as e:
462 raise FDroidBuildVmException("could not cerate snapshot '%s' "
464 % (snapshot_name, self.srvname)) from e
466 def snapshot_list(self):
469 dom = self.conn.lookupByName(self.srvname)
470 return dom.listAllSnapshots()
471 except libvirt.libvirtError as e:
472 raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e
474 def snapshot_exists(self, snapshot_name):
477 dom = self.conn.lookupByName(self.srvname)
478 return dom.snapshotLookupByName(snapshot_name) is not None
479 except libvirt.libvirtError:
482 def snapshot_revert(self, snapshot_name):
483 logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
486 dom = self.conn.lookupByName(self.srvname)
487 snap = dom.snapshotLookupByName(snapshot_name)
488 dom.revertToSnapshot(snap)
489 except libvirt.libvirtError as e:
490 raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\''
491 % (self.srvname, snapshot_name)) from e
494 class VirtualboxBuildVm(FDroidBuildVm):
496 def __init__(self, srvdir):
497 self.provider = 'virtualbox'
498 super().__init__(srvdir)
500 def snapshot_create(self, snapshot_name):
501 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
503 _check_call(['VBoxManage', 'snapshot', self.srvuuid, 'take', 'fdroidclean'], cwd=self.srvdir)
504 except subprocess.CalledProcessError as e:
505 raise FDroidBuildVmException('could not cerate snapshot '
506 'of virtualbox vm %s'
507 % self.srvname) from e
509 def snapshot_list(self):
511 o = _check_output(['VBoxManage', 'snapshot',
512 self.srvuuid, 'list',
513 '--details'], cwd=self.srvdir)
515 except subprocess.CalledProcessError as e:
516 raise FDroidBuildVmException("could not list snapshots "
517 "of virtualbox vm '%s'"
518 % (self.srvname)) from e
520 def snapshot_exists(self, snapshot_name):
522 return str(snapshot_name) in str(self.snapshot_list())
523 except FDroidBuildVmException:
526 def snapshot_revert(self, snapshot_name):
527 logger.info("reverting vm '%s' to snapshot '%s'",
528 self.srvname, snapshot_name)
530 _check_call(['VBoxManage', 'snapshot', self.srvuuid,
531 'restore', 'fdroidclean'], cwd=self.srvdir)
532 except subprocess.CalledProcessError as e:
533 raise FDroidBuildVmException("could not load snapshot "
534 "'fdroidclean' for vm '%s'"
535 % (self.srvname)) from e