import tarfile
import traceback
import time
-import json
import requests
import tempfile
import textwrap
pass
-def get_vm_provider():
- """Determine vm provider based on .vagrant directory content
- """
- if os.path.exists(os.path.join('builder', '.vagrant', 'machines',
- 'default', 'libvirt')):
- return 'libvirt'
- return 'virtualbox'
-
-
-def vm_get_builder_id(provider):
- vd = os.path.join('builder', '.vagrant')
- if os.path.isdir(vd):
- # Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
- with open(os.path.join(vd, 'machines', 'default',
- provider, 'id')) as vf:
- id = vf.read()
- return id
- else:
- # Vagrant 1.0 - it's a json file...
- with open(os.path.join('builder', '.vagrant')) as vf:
- v = json.load(vf)
- return v['active']['default']
-
-
-def vm_get_builder_status():
- """Get the current status of builder vm.
-
- :returns: one of: 'running', 'paused', 'shutoff', 'not created'
- If something is wrong with vagrant or the vm 'unknown' is returned.
- """
- (ret, out) = vagrant(['status'], cwd='builder')
-
- allowed_providers = 'virtualbox|libvirt'
- allowed_states = 'running|paused|shutoff|not created'
-
- r = re.compile('^\s*(?P<vagrant_name>\w+)\s+' +
- '(?P<vm_state>' + allowed_states + ')' +
- '\s+\((?P<provider>' + allowed_providers + ')\)\s*$')
-
- for line in out.split('\n'):
- m = r.match(line)
- if m:
- s = m.group('vm_state')
- if options.verbose:
- logging.debug('current builder vm status: ' + s)
- return s
- if options.verbose:
- logging.debug('current builder vm status: unknown')
- return 'unknown'
-
-
-def vm_is_builder_valid(provider):
- """Returns True if we have a valid-looking builder vm
- """
- if not os.path.exists(os.path.join('builder', 'Vagrantfile')):
- return False
- vd = os.path.join('builder', '.vagrant')
- if not os.path.exists(vd):
- return False
- if not os.path.isdir(vd):
- # Vagrant 1.0 - if the directory is there, it's valid...
- return True
- # Vagrant 1.2 - the directory can exist, but the id can be missing...
- if not os.path.exists(os.path.join(vd, 'machines', 'default',
- provider, 'id')):
- return False
- return True
-
-
-def vagrant(params, cwd=None, printout=False):
- """Run a vagrant command.
-
- :param: list of parameters to pass to vagrant
- :cwd: directory to run in, or None for current directory
- :printout: has not effect
- :returns: (ret, out) where ret is the return code, and out
- is the stdout (and stderr) from vagrant
- """
- p = FDroidPopen(['vagrant'] + params, cwd=cwd, output=printout, stderr_to_stdout=printout)
- return (p.returncode, p.output)
-
-
-def get_vagrant_sshinfo():
- """Get ssh connection info for a vagrant VM
-
- :returns: A dictionary containing 'hostname', 'port', 'user'
- and 'idfile'
- """
- if subprocess.call('vagrant ssh-config >sshconfig',
- cwd='builder', shell=True) != 0:
- raise BuildException("Error getting ssh config")
- vagranthost = 'default' # Host in ssh config file
- sshconfig = paramiko.SSHConfig()
- sshf = open(os.path.join('builder', 'sshconfig'), 'r')
- sshconfig.parse(sshf)
- sshf.close()
- sshconfig = sshconfig.lookup(vagranthost)
- idfile = sshconfig['identityfile']
- if isinstance(idfile, list):
- idfile = idfile[0]
- elif idfile.startswith('"') and idfile.endswith('"'):
- idfile = idfile[1:-1]
- return {'hostname': sshconfig['hostname'],
- 'port': int(sshconfig['port']),
- 'user': sshconfig['user'],
- 'idfile': idfile}
-
-
-def vm_shutdown_builder():
- """Turn off builder vm.
- """
- if options.server:
- if os.path.exists(os.path.join('builder', 'Vagrantfile')):
- vagrant(['halt'], cwd='builder')
-
-
-def vm_snapshot_list(provider):
- output = options.verbose
- if provider is 'virtualbox':
- p = FDroidPopen(['VBoxManage', 'snapshot',
- vm_get_builder_id(provider), 'list',
- '--details'], cwd='builder',
- output=output, stderr_to_stdout=output)
- elif provider is 'libvirt':
- p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list',
- vm_get_builder_id(provider)],
- output=output, stderr_to_stdout=output)
- return p.output
-
-
-def vm_snapshot_clean_available(provider):
- return 'fdroidclean' in vm_snapshot_list(provider)
-
-
-def vm_snapshot_restore(provider):
- """Does a rollback of the build vm.
- """
- output = options.verbose
- if provider is 'virtualbox':
- p = FDroidPopen(['VBoxManage', 'snapshot',
- vm_get_builder_id(provider), 'restore',
- 'fdroidclean'], cwd='builder',
- output=output, stderr_to_stdout=output)
- elif provider is 'libvirt':
- p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert',
- vm_get_builder_id(provider), 'fdroidclean'],
- output=output, stderr_to_stdout=output)
- return p.returncode == 0
-
-
-def vm_snapshot_create(provider):
- output = options.verbose
- if provider is 'virtualbox':
- p = FDroidPopen(['VBoxManage', 'snapshot',
- vm_get_builder_id(provider),
- 'take', 'fdroidclean'], cwd='builder',
- output=output, stderr_to_stdout=output)
- elif provider is 'libvirt':
- p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as',
- vm_get_builder_id(provider), 'fdroidclean'],
- output=output, stderr_to_stdout=output)
- return p.returncode != 0
-
-
-def vm_test_ssh_into_builder():
- logging.info("Connecting to virtual machine...")
- sshinfo = get_vagrant_sshinfo()
- sshs = paramiko.SSHClient()
- sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
- port=sshinfo['port'], timeout=300,
- look_for_keys=False,
- key_filename=sshinfo['idfile'])
- sshs.close()
-
-
def vm_new_get_clean_builder(serverdir, reset=False):
if not os.path.isdir(serverdir):
if os.path.islink(serverdir):
return sshinfo
-def vm_get_clean_builder(reset=False):
- """Get a clean VM ready to do a buildserver build.
-
- This might involve creating and starting a new virtual machine from
- scratch, or it might be as simple (unless overridden by the reset
- parameter) as re-using a snapshot created previously.
-
- A BuildException will be raised if anything goes wrong.
-
- :reset: True to force creating from scratch.
- :returns: A dictionary containing 'hostname', 'port', 'user'
- and 'idfile'
- """
- provider = get_vm_provider()
-
- # Reset existing builder machine to a clean state if possible.
- vm_ok = False
- if not reset:
- logging.info("Checking for valid existing build server")
- if vm_is_builder_valid(provider):
- logging.info("...VM is present (%s)" % provider)
- if vm_snapshot_clean_available(provider):
- logging.info("...snapshot exists - resetting build server to " +
- "clean state")
- status = vm_get_builder_status()
- if status == 'running':
- vm_test_ssh_into_builder()
- logging.info("...suspending builder vm")
- vagrant(['suspend'], cwd='builder')
- logging.info("...waiting a sec...")
- time.sleep(10)
- elif status == 'shutoff':
- logging.info('...starting builder vm')
- vagrant(['up'], cwd='builder')
- logging.info('...waiting a sec...')
- time.sleep(10)
- vm_test_ssh_into_builder()
- logging.info('...suspending builder vm')
- vagrant(['suspend'], cwd='builder')
- logging.info("...waiting a sec...")
- time.sleep(10)
- if options.verbose:
- vm_get_builder_status()
-
- if vm_snapshot_restore(provider):
- if options.verbose:
- vm_get_builder_status()
- logging.info("...reset to snapshot - server is valid")
- retcode, output = vagrant(['up'], cwd='builder')
- if retcode != 0:
- raise BuildException("Failed to start build server")
- logging.info("...waiting a sec...")
- time.sleep(10)
- sshinfo = get_vagrant_sshinfo()
- vm_ok = True
- else:
- logging.info("...failed to reset to snapshot")
- else:
- logging.info("...snapshot doesn't exist - "
- "VBoxManage snapshot list:\n" +
- vm_snapshot_list(provider))
- else:
- logging.info('...VM not present')
-
- # If we can't use the existing machine for any reason, make a
- # new one from scratch.
- if not vm_ok:
- if os.path.isdir('builder'):
- vm = vmtools.get_build_vm('builder')
- vm.destroy()
- else:
- os.mkdir('builder')
-
- p = subprocess.Popen(['vagrant', '--version'],
- universal_newlines=True,
- stdout=subprocess.PIPE)
- vver = p.communicate()[0].strip().split(' ')[1]
- if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
- raise BuildException("Unsupported vagrant version {0}".format(vver))
-
- with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
- vf.write('Vagrant.configure("2") do |config|\n')
- vf.write(' config.vm.box = "buildserver"\n')
- vf.write(' config.vm.synced_folder ".", "/vagrant", disabled: true\n')
- vf.write('end\n')
-
- logging.info("Starting new build server")
- retcode, _ = vagrant(['up'], cwd='builder')
- if retcode != 0:
- raise BuildException("Failed to start build server")
- provider = get_vm_provider()
- sshinfo = get_vagrant_sshinfo()
-
- # Open SSH connection to make sure it's working and ready...
- vm_test_ssh_into_builder()
-
- logging.info("Saving clean state of new build server")
- retcode, _ = vagrant(['suspend'], cwd='builder')
- if retcode != 0:
- raise BuildException("Failed to suspend build server")
- logging.info("...waiting a sec...")
- time.sleep(10)
- if vm_snapshot_create(provider):
- raise BuildException("Failed to take snapshot")
- logging.info("...waiting a sec...")
- time.sleep(10)
- logging.info("Restarting new build server")
- retcode, _ = vagrant(['up'], cwd='builder')
- if retcode != 0:
- raise BuildException("Failed to start build server")
- logging.info("...waiting a sec...")
- time.sleep(10)
- # Make sure it worked...
- if not vm_snapshot_clean_available(provider):
- raise BuildException("Failed to take snapshot.")
-
- return sshinfo
-
-
-def vm_suspend_builder():
- """Release the VM previously started with vm_get_clean_builder().
-
- This should always be called after each individual app build attempt.
- """
- logging.info("Suspending build server")
- subprocess.call(['vagrant', 'suspend'], cwd='builder')
-
-
# Note that 'force' here also implies test mode.
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
"""Do a build on the builder vm.
~ianmdlvl / fdroidserver.git / commitdiff