3 # vmtools.py - part of the FDroid server tools
4 # Copyright (C) 2017 Michael Poehn <michael.poehn@fsfe.org>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from os import remove as rmfile
20 from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser
28 from .common import FDroidException
29 from logging import getLogger
31 from fdroidserver import _
34 lock = threading.Lock()
36 logger = getLogger('fdroidserver-vmtools')
39 def get_clean_builder(serverdir, reset=False):
40 if not os.path.isdir(serverdir):
41 if os.path.islink(serverdir):
43 logger.info("buildserver path does not exists, creating %s", serverdir)
44 os.makedirs(serverdir)
45 vagrantfile = os.path.join(serverdir, 'Vagrantfile')
46 if not os.path.isfile(vagrantfile):
47 with open(os.path.join('builder', 'Vagrantfile'), 'w') as f:
48 f.write(textwrap.dedent("""\
49 # generated file, do not change.
51 Vagrant.configure("2") do |config|
52 config.vm.box = "buildserver"
53 config.vm.synced_folder ".", "/vagrant", disabled: true
56 vm = get_build_vm(serverdir)
58 logger.info('resetting buildserver by request')
59 elif not vm.vagrant_uuid_okay():
60 logger.info('resetting buildserver, because vagrant vm is not okay.')
62 elif not vm.snapshot_exists('fdroidclean'):
63 logger.info("resetting buildserver, because snapshot 'fdroidclean' is not present.")
72 logger.info('buildserver recreated: taking a clean snapshot')
73 vm.snapshot_create('fdroidclean')
75 logger.info('builserver ok: reverting to clean snapshot')
76 vm.snapshot_revert('fdroidclean')
80 sshinfo = vm.sshinfo()
81 except FDroidBuildVmException:
82 # workaround because libvirt sometimes likes to forget
83 # about ssh connection info even thou the vm is running
86 sshinfo = vm.sshinfo()
91 def _check_call(cmd, shell=False, cwd=None):
92 logger.debug(' '.join(cmd))
93 return subprocess.check_call(cmd, shell=shell, cwd=cwd)
96 def _check_output(cmd, shell=False, cwd=None):
97 logger.debug(' '.join(cmd))
98 return subprocess.check_output(cmd, shell=shell, cwd=cwd)
101 def get_build_vm(srvdir, provider=None):
102 """Factory function for getting FDroidBuildVm instances.
104 This function tries to figure out what hypervisor should be used
105 and creates an object for controlling a build VM.
107 :param srvdir: path to a directory which contains a Vagrantfile
108 :param provider: optionally this parameter allows specifiying an
109 spesific vagrant provider.
110 :returns: FDroidBuildVm instance.
112 abssrvdir = abspath(srvdir)
114 # use supplied provider
116 if provider == 'libvirt':
117 logger.debug('build vm provider \'libvirt\' selected')
118 return LibvirtBuildVm(abssrvdir)
119 elif provider == 'virtualbox':
120 logger.debug('build vm provider \'virtualbox\' selected')
121 return VirtualboxBuildVm(abssrvdir)
123 logger.warn('build vm provider not supported: \'%s\'', provider)
125 # try guessing provider from installed software
127 kvm_installed = 0 == _check_call(['which', 'kvm'])
128 except subprocess.CalledProcessError:
129 kvm_installed = False
131 kvm_installed |= 0 == _check_call(['which', 'qemu'])
132 except subprocess.CalledProcessError:
135 vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'])
136 except subprocess.CalledProcessError:
137 vbox_installed = False
138 if kvm_installed and vbox_installed:
139 logger.debug('both kvm and vbox are installed.')
141 logger.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'')
142 return LibvirtBuildVm(abssrvdir)
144 logger.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'')
145 return VirtualboxBuildVm(abssrvdir)
147 logger.debug('could not confirm that either virtualbox or kvm/libvirt are installed')
149 # try guessing provider from .../srvdir/.vagrant internals
150 has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant',
151 'machines', 'default', 'libvirt'))
152 has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant',
153 'machines', 'default', 'virtualbox'))
154 if has_libvirt_machine and has_vbox_machine:
155 logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
156 return VirtualboxBuildVm(abssrvdir)
157 elif has_libvirt_machine:
158 logger.debug('build vm provider lookup found \'libvirt\'')
159 return LibvirtBuildVm(abssrvdir)
160 elif has_vbox_machine:
161 logger.debug('build vm provider lookup found \'virtualbox\'')
162 return VirtualboxBuildVm(abssrvdir)
164 logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'')
165 return VirtualboxBuildVm(abssrvdir)
168 class FDroidBuildVmException(FDroidException):
172 class FDroidBuildVm():
173 """Abstract base class for working with FDroids build-servers.
175 Use the factory method `fdroidserver.vmtools.get_build_vm()` for
176 getting correct instances of this class.
178 This is intended to be a hypervisor independant, fault tolerant
179 wrapper around the vagrant functions we use.
181 def __init__(self, srvdir):
182 """Create new server class.
185 self.srvname = basename(srvdir) + '_default'
186 self.vgrntfile = joinpath(srvdir, 'Vagrantfile')
187 self.srvuuid = self._vagrant_fetch_uuid()
188 if not isdir(srvdir):
189 raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
190 if not isfile(self.vgrntfile):
191 raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
193 self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
195 def up(self, provision=True):
199 self.vgrnt.up(provision=provision)
200 self.srvuuid = self._vagrant_fetch_uuid()
201 except subprocess.CalledProcessError as e:
202 raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
207 logger.info('suspending buildserver')
210 except subprocess.CalledProcessError as e:
211 raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
216 self.vgrnt.halt(force=True)
219 """Remove every trace of this VM from the system.
221 This includes deleting:
222 * hypervisor specific definitions
223 * vagrant state informations (eg. `.vagrant` folder)
224 * images related to this vm
226 logger.info("destroying vm '%s'", self.srvname)
229 logger.debug('vagrant destroy completed')
230 except subprocess.CalledProcessError as e:
231 logger.exception('vagrant destroy failed: %s', e)
232 vgrntdir = joinpath(self.srvdir, '.vagrant')
234 shutil.rmtree(vgrntdir)
235 logger.debug('deleted vagrant dir: %s', vgrntdir)
236 except Exception as e:
237 logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e)
239 _check_call(['vagrant', 'global-status', '--prune'])
240 except subprocess.CalledProcessError as e:
241 logger.debug('pruning global vagrant status failed: %s', e)
243 def package(self, output=None):
244 self.vgrnt.package(output=output)
246 def vagrant_uuid_okay(self):
247 '''Having an uuid means that vagrant up has run successfully.'''
248 if self.srvuuid is None:
252 def _vagrant_file_name(self, name):
253 return name.replace('/', '-VAGRANTSLASH-')
255 def _vagrant_fetch_uuid(self):
256 if isfile(joinpath(self.srvdir, '.vagrant')):
257 # Vagrant 1.0 - it's a json file...
258 with open(joinpath(self.srvdir, '.vagrant')) as f:
259 id = json.load(f)['active']['default']
260 logger.debug('vm uuid: %s', id)
262 elif isfile(joinpath(self.srvdir, '.vagrant', 'machines',
263 'default', self.provider, 'id')):
264 # Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
265 with open(joinpath(self.srvdir, '.vagrant', 'machines',
266 'default', self.provider, 'id')) as f:
268 logger.debug('vm uuid: %s', id)
271 logger.debug('vm uuid is None')
274 def box_add(self, boxname, boxfile, force=True):
275 """Add vagrant box to vagrant.
277 :param boxname: name assigned to local deployment of box
278 :param boxfile: path to box file
279 :param force: overwrite existing box image (default: True)
281 boxfile = abspath(boxfile)
282 if not isfile(boxfile):
283 raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile)
284 self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
286 def box_remove(self, boxname):
288 _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
289 except subprocess.CalledProcessError as e:
290 logger.debug('tried removing box %s, but is did not exist: %s', boxname, e)
291 boxpath = joinpath(expanduser('~'), '.vagrant',
292 self._vagrant_file_name(boxname))
294 logger.info("attempting to remove box '%s' by deleting: %s",
296 shutil.rmtree(boxpath)
299 """Get ssh connection info for a vagrant VM
301 :returns: A dictionary containing 'hostname', 'port', 'user'
306 _check_call(['vagrant ssh-config > sshconfig'],
307 cwd=self.srvdir, shell=True)
308 vagranthost = 'default' # Host in ssh config file
309 sshconfig = paramiko.SSHConfig()
310 with open(joinpath(self.srvdir, 'sshconfig'), 'r') as f:
312 sshconfig = sshconfig.lookup(vagranthost)
313 idfile = sshconfig['identityfile']
314 if isinstance(idfile, list):
316 elif idfile.startswith('"') and idfile.endswith('"'):
317 idfile = idfile[1:-1]
318 return {'hostname': sshconfig['hostname'],
319 'port': int(sshconfig['port']),
320 'user': sshconfig['user'],
322 except subprocess.CalledProcessError as e:
323 raise FDroidBuildVmException("Error getting ssh config") from e
325 def snapshot_create(self, snapshot_name):
326 raise NotImplementedError('not implemented, please use a sub-type instance')
328 def snapshot_list(self):
329 raise NotImplementedError('not implemented, please use a sub-type instance')
331 def snapshot_exists(self, snapshot_name):
332 raise NotImplementedError('not implemented, please use a sub-type instance')
334 def snapshot_revert(self, snapshot_name):
335 raise NotImplementedError('not implemented, please use a sub-type instance')
338 class LibvirtBuildVm(FDroidBuildVm):
339 def __init__(self, srvdir):
340 self.provider = 'libvirt'
341 super().__init__(srvdir)
345 self.conn = libvirt.open('qemu:///system')
346 except libvirt.libvirtError as e:
347 raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e))
353 # resorting to virsh instead of libvirt python bindings, because
354 # this is way more easy and therefore fault tolerant.
355 # (eg. lookupByName only works on running VMs)
357 _check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname))
358 except subprocess.CalledProcessError as e:
359 logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e)
361 # libvirt python bindings do not support all flags required
362 # for undefining domains correctly.
363 _check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
364 except subprocess.CalledProcessError as e:
365 logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e)
367 def package(self, output=None, keep_box_file=False):
369 output = "buildserver.box"
370 logger.debug('no output name set for packaging \'%s\',' +
371 'defaulting to %s', self.srvname, output)
372 storagePool = self.conn.storagePoolLookupByName('default')
373 domainInfo = self.conn.lookupByName(self.srvname).info()
376 if isfile('metadata.json'):
377 rmfile('metadata.json')
378 if isfile('Vagrantfile'):
379 rmfile('Vagrantfile')
380 if isfile('box.img'):
383 logger.debug('preparing box.img for box %s', output)
384 vol = storagePool.storageVolLookupByName(self.srvname + '.img')
385 imagepath = vol.path()
386 # TODO use a libvirt storage pool to ensure the img file is readable
387 if not os.access(imagepath, os.R_OK):
388 logger.warning(_('Cannot read "{path}"!').format(path=imagepath))
389 _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
390 shutil.copy2(imagepath, 'box.img')
391 _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
392 img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
393 img_info = json.loads(img_info_raw.decode('utf-8'))
394 metadata = {"provider": "libvirt",
395 "format": img_info['format'],
396 "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
399 logger.debug('preparing metadata.json for box %s', output)
400 with open('metadata.json', 'w') as fp:
401 fp.write(json.dumps(metadata))
402 logger.debug('preparing Vagrantfile for box %s', output)
403 vagrantfile = textwrap.dedent("""\
404 Vagrant.configure("2") do |config|
405 config.ssh.username = "vagrant"
406 config.ssh.password = "vagrant"
408 config.vm.provider :libvirt do |libvirt|
410 libvirt.driver = "kvm"
412 libvirt.connect_via_ssh = false
413 libvirt.storage_pool_name = "default"
414 libvirt.cpus = {cpus}
415 libvirt.memory = {memory}
418 end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])}))
419 with open('Vagrantfile', 'w') as fp:
420 fp.write(vagrantfile)
421 with tarfile.open(output, 'w:gz') as tar:
422 logger.debug('adding metadata.json to box %s ...', output)
423 tar.add('metadata.json')
424 logger.debug('adding Vagrantfile to box %s ...', output)
425 tar.add('Vagrantfile')
426 logger.debug('adding box.img to box %s ...', output)
429 if not keep_box_file:
430 logger.debug('box packaging complete, removing temporary files.')
431 rmfile('metadata.json')
432 rmfile('Vagrantfile')
436 logger.warn('could not connect to storage-pool \'default\',' +
437 'skipping packaging buildserver box')
439 def box_add(self, boxname, boxfile, force=True):
440 boximg = '%s_vagrant_box_image_0.img' % (boxname)
443 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', boximg])
444 logger.debug("removed old box image '%s' from libvirt storeage pool", boximg)
445 except subprocess.CalledProcessError as e:
446 logger.debug("tired removing old box image '%s', file was not present in first place", boximg, exc_info=e)
447 super().box_add(boxname, boxfile, force)
449 def box_remove(self, boxname):
450 super().box_remove(boxname)
452 _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)])
453 except subprocess.CalledProcessError as e:
454 logger.debug("tired removing '%s', file was not present in first place", boxname, exc_info=e)
456 def snapshot_create(self, snapshot_name):
457 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
459 _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name])
460 except subprocess.CalledProcessError as e:
461 raise FDroidBuildVmException("could not cerate snapshot '%s' "
463 % (snapshot_name, self.srvname)) from e
465 def snapshot_list(self):
468 dom = self.conn.lookupByName(self.srvname)
469 return dom.listAllSnapshots()
470 except libvirt.libvirtError as e:
471 raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e
473 def snapshot_exists(self, snapshot_name):
476 dom = self.conn.lookupByName(self.srvname)
477 return dom.snapshotLookupByName(snapshot_name) is not None
478 except libvirt.libvirtError:
481 def snapshot_revert(self, snapshot_name):
482 logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
485 dom = self.conn.lookupByName(self.srvname)
486 snap = dom.snapshotLookupByName(snapshot_name)
487 dom.revertToSnapshot(snap)
488 except libvirt.libvirtError as e:
489 raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\''
490 % (self.srvname, snapshot_name)) from e
493 class VirtualboxBuildVm(FDroidBuildVm):
495 def __init__(self, srvdir):
496 self.provider = 'virtualbox'
497 super().__init__(srvdir)
499 def snapshot_create(self, snapshot_name):
500 logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
502 _check_call(['VBoxManage', 'snapshot', self.srvuuid, 'take', 'fdroidclean'], cwd=self.srvdir)
503 except subprocess.CalledProcessError as e:
504 raise FDroidBuildVmException('could not cerate snapshot '
505 'of virtualbox vm %s'
506 % self.srvname) from e
508 def snapshot_list(self):
510 o = _check_output(['VBoxManage', 'snapshot',
511 self.srvuuid, 'list',
512 '--details'], cwd=self.srvdir)
514 except subprocess.CalledProcessError as e:
515 raise FDroidBuildVmException("could not list snapshots "
516 "of virtualbox vm '%s'"
517 % (self.srvname)) from e
519 def snapshot_exists(self, snapshot_name):
521 return str(snapshot_name) in str(self.snapshot_list())
522 except FDroidBuildVmException:
525 def snapshot_revert(self, snapshot_name):
526 logger.info("reverting vm '%s' to snapshot '%s'",
527 self.srvname, snapshot_name)
529 _check_call(['VBoxManage', 'snapshot', self.srvuuid,
530 'restore', 'fdroidclean'], cwd=self.srvdir)
531 except subprocess.CalledProcessError as e:
532 raise FDroidBuildVmException("could not load snapshot "
533 "'fdroidclean' for vm '%s'"
534 % (self.srvname)) from e