import json
from ConfigParser import ConfigParser
from optparse import OptionParser, OptionError
+import logging
import common, metadata
from common import BuildException, VCSException, FDroidPopen
:returns: (ret, out) where ret is the return code, and out
is the stdout (and stderr) from vagrant
"""
- p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- out = p.communicate()[0]
- if options.verbose:
- print out
- return (p.returncode, out)
+ p = FDroidPopen(['vagrant'] + params, cwd=cwd)
+ return (p.returncode, p.stdout)
# Note that 'force' here also implies test mode.
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not options.resetserver:
- print "Checking for valid existing build server"
+ logging.info("Checking for valid existing build server")
+
if got_valid_builder_vm():
- print "...VM is present"
- p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
- cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = p.communicate()[0]
- if 'fdroidclean' in output:
- if options.verbose:
- print "...snapshot exists - resetting build server to clean state"
+ logging.info("...VM is present")
+ p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'], cwd='builder')
+ if 'fdroidclean' in p.stdout:
+ logging.info("...snapshot exists - resetting build server to clean state")
retcode, output = vagrant(['status'], cwd='builder')
+
if 'running' in output:
- if options.verbose:
- print "...suspending"
+ logging.info("...suspending")
vagrant(['suspend'], cwd='builder')
- print "...waiting a sec..."
+ logging.info("...waiting a sec...")
time.sleep(10)
- p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean'],
- cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = p.communicate()[0]
- if options.verbose:
- print output
+ p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean'],
+ cwd='builder')
+
if p.returncode == 0:
- print "...reset to snapshot - server is valid"
+ logging.info("...reset to snapshot - server is valid")
retcode, output = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
- print "...waiting a sec..."
+ logging.info("...waiting a sec...")
time.sleep(10)
vm_ok = True
else:
- print "...failed to reset to snapshot"
+ logging.info("...failed to reset to snapshot")
else:
- print "...snapshot doesn't exist - VBoxManage snapshot list:\n" + output
+ logging.info("...snapshot doesn't exist - VBoxManage snapshot list:\n" + output)
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
- print "Removing broken/incomplete/unwanted build server"
+ logging.info("Removing broken/incomplete/unwanted build server")
vagrant(['destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
- p = subprocess.Popen('vagrant --version', shell=True, stdout=subprocess.PIPE)
- vver = p.communicate()[0]
- if vver.startswith('Vagrant version 1.2'):
+ p = FDroidPopen('vagrant --version', shell=True, stdout=subprocess.PIPE)
+ if p.stdout.startswith('Vagrant version 1.2'):
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant.configure("2") do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('end\n')
- print "Starting new build server"
+ logging.info("Starting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
- print "Connecting to virtual machine..."
+ logging.info("Connecting to virtual machine...")
if subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True) != 0:
raise BuildException("Error getting ssh config")
key_filename=idfile)
sshs.close()
- print "Saving clean state of new build server"
+ logging.info("Saving clean state of new build server")
retcode, _ = vagrant(['suspend'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to suspend build server")
- print "...waiting a sec..."
+ logging.info("...waiting a sec...")
time.sleep(10)
- p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean'],
- cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = p.communicate()[0]
+ p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean'],
+ cwd='builder')
if p.returncode != 0:
- print output
raise BuildException("Failed to take snapshot")
- print "...waiting a sec..."
+ logging.info("...waiting a sec...")
time.sleep(10)
- print "Restarting new build server"
+ logging.info("Restarting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
- print "...waiting a sec..."
+ logging.info("...waiting a sec...")
time.sleep(10)
# Make sure it worked...
- p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
- cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = p.communicate()[0]
- if 'fdroidclean' not in output:
+ p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
+ cwd='builder')
+ if 'fdroidclean' not in p.stdout:
raise BuildException("Failed to take snapshot.")
try:
# Get SSH configuration settings for us to connect...
- if options.verbose:
- print "Getting ssh configuration..."
+ logging.info("Getting ssh configuration...")
subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True)
vagranthost = 'default' # Host in ssh config file
sshconfig = sshconfig.lookup(vagranthost)
# Open SSH connection...
- if options.verbose:
- print "Connecting to virtual machine..."
+ logging.info("Connecting to virtual machine...")
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
ftp.chdir('..')
ftp.chdir('..')
- print "Preparing server for build..."
+ logging.info("Preparing server for build...")
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths:
- if options.verbose:
- print "Sending srclib '" + lib + "'"
+ logging.info("Sending srclib '%s'" % lib)
ftp.chdir('/home/vagrant/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
send_dir(build_dir)
# Execute the build script...
- print "Starting build..."
+ logging.info("Starting build...")
chan = sshs.get_transport().open_session()
chan.get_pty()
cmdline = 'python build.py --on-server'
while chan.recv_ready():
output += chan.recv(1024)
time.sleep(0.1)
- print "...getting exit status"
+ logging.info("...getting exit status")
returncode = chan.recv_exit_status()
while True:
get = chan.recv(1024)
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output)
# Retrieve the built files...
- print "Retrieving build output..."
+ logging.info("Retrieving build output...")
if force:
ftp.chdir('/home/vagrant/tmp')
else:
tarball = common.getsrcname(app,thisbuild)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
- ftp.get(tarball, os.path.join(output_dir, tarball))
+ if not options.notarball:
+ ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException("Build failed for %s:%s - missing output files" % (app['id'], thisbuild['version']), output)
ftp.close()
finally:
# Suspend the build server.
- print "Suspending build server"
+ logging.info("Suspending build server")
subprocess.call(['vagrant', 'suspend'], cwd='builder')
def adapt_gradle(build_dir):
for root, dirs, files in os.walk(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
- if options.verbose:
- print "Adapting build.gradle at %s" % path
+ logging.info("Adapting build.gradle at %s" % path)
subprocess.call(['sed', '-i',
r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"'
# different from the default ones
p = None
if thisbuild['type'] == 'maven':
- print "Cleaning Maven project..."
+ logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']:
elif thisbuild['type'] == 'gradle':
- print "Cleaning Gradle project..."
+ logging.info("Cleaning Gradle project...")
cmd = [config['gradle'], 'clean']
if '@' in thisbuild['gradle']:
pass
elif thisbuild['type'] == 'ant':
- print "Cleaning Ant project..."
+ logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
(app['id'], thisbuild['version']), p.stdout)
# Scan before building...
- print "Scanning source for common problems..."
+ logging.info("Scanning source for common problems...")
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
- print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
+ logging.info('Scanner found %d problems:' % len(buildprobs))
for problem in buildprobs:
- print ' %s' % problem
+ logging.info(' %s' % problem)
if not force:
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
- # Build the source tarball right before we build the release...
- logging.info("Creating source tarball...")
- tarname = common.getsrcname(app,thisbuild)
- tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
- def tarexc(f):
- return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
- tarball.add(build_dir, tarname, exclude=tarexc)
- tarball.close()
+ if not options.notarball:
+ # Build the source tarball right before we build the release...
- print "Creating source tarball..."
++ logging.info("Creating source tarball...")
+ tarname = common.getsrcname(app,thisbuild)
+ tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
+ def tarexc(f):
+ return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
+ tarball.add(build_dir, tarname, exclude=tarexc)
+ tarball.close()
# Run a build command if one is required...
if 'build' in thisbuild:
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
- if options.verbose:
- print "Running 'build' commands in %s" % root_dir
+ logging.info("Running 'build' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
# Build native stuff if required...
if thisbuild.get('buildjni') not in (None, 'no'):
- print "Building native libraries..."
+ logging.info("Building native libraries...")
jni_components = thisbuild.get('buildjni')
if jni_components == 'yes':
jni_components = ['']
jni_components = [c.strip() for c in jni_components.split(';')]
ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
for d in jni_components:
- if options.verbose:
- print "Running ndk-build in " + root_dir + '/' + d
+ logging.info("Building native code in '%s'" % d)
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
p = None
# Build the release...
if thisbuild['type'] == 'maven':
- print "Building Maven project..."
+ logging.info("Building Maven project...")
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy':
- print "Building Kivy project..."
+ logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle':
- print "Building Gradle project..."
+ logging.info("Building Gradle project...")
if '@' in thisbuild['gradle']:
flavours = thisbuild['gradle'].split('@')[0].split(',')
gradle_dir = thisbuild['gradle'].split('@')[1]
p = FDroidPopen(commands, cwd=gradle_dir)
- else:
+ elif thisbuild['type'] == 'ant':
- print "Building Ant project..."
+ logging.info("Building Ant project...")
cmd = ['ant']
if 'antcommand' in thisbuild:
cmd += [thisbuild['antcommand']]
bindir = os.path.join(root_dir, 'bin')
- if p.returncode != 0:
+ if p is not None and p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout)
- print "Successfully built version " + thisbuild['version'] + ' of ' + app['id']
+ logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id'])
if thisbuild['type'] == 'maven':
stdout_apk = '\n'.join([
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
+ elif thisbuild['type'] == 'raw':
+ src = os.path.join(root_dir, thisbuild['output'])
+ src = os.path.normpath(src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
- print "Checking " + src
+ logging.info("Checking " + src)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
- p = subprocess.Popen([os.path.join(config['sdk_path'],
+ p = FDroidPopen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
- 'dump', 'badging', src],
- stdout=subprocess.PIPE)
- output = p.communicate()[0]
+ 'dump', 'badging', src], output=False)
vercode = None
version = None
foundid = None
- for line in output.splitlines():
+ for line in p.stdout.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
m = pat.match(line)
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
- if output_dir != tmp_dir:
+ if output_dir != tmp_dir and not options.notarball:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
if 'disable' in thisbuild:
return False
- print "Building version " + thisbuild['version'] + ' of ' + app['id']
+ logging.info("Building version " + thisbuild['version'] + ' of ' + app['id'])
if server:
# When using server mode, still keep a local cache of the repo, by
help="Reset and create a brand new build server, even if the existing one appears to be ok.")
parser.add_option("--on-server", dest="onserver", action="store_true", default=False,
help="Specify that we're running on the build server")
+ parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False,
+ help="Don't create a source tarball, useful when testing a build")
parser.add_option("-f", "--force", action="store_true", default=False,
help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
parser.add_option("-a", "--all", action="store_true", default=False,
log_dir = 'logs'
if not os.path.isdir(log_dir):
- print "Creating log directory"
+ logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
- print "Creating temporary directory"
+ logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
if options.test:
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
- print "Creating output directory"
+ logging.info("Creating output directory")
os.makedirs(output_dir)
if config['archive_older'] != 0:
build_dir = 'build'
if not os.path.isdir(build_dir):
- print "Creating build directory"
+ logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
build_dir = os.path.join('build', app['id'])
# Set up vcs interface and make sure we have the latest code...
- if options.verbose:
- print "Getting {0} vcs interface for {1}".format(
- app['Repo Type'], app['Repo'])
+ logging.info("Getting {0} vcs interface for {1}".format(
+ app['Repo Type'], app['Repo']))
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
first = False
- if options.verbose:
- print "Checking " + thisbuild['version']
+ logging.info("Checking " + thisbuild['version'])
if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test,
options.server, options.force, options.onserver):
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
- print "Could not build app %s due to BuildException: %s" % (app['id'], be)
+ logging.info("Could not build app %s due to BuildException: %s" % (app['id'], be))
if options.stop:
sys.exit(1)
failed_apps[app['id']] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
- print "VCS error while building app %s: %s" % (app['id'], vcse)
+ logging.info("VCS error while building app %s: %s" % (app['id'], vcse))
if options.stop:
sys.exit(1)
failed_apps[app['id']] = vcse
wikilog = str(vcse)
except Exception as e:
- print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+ logging.info("Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()))
if options.stop:
sys.exit(1)
failed_apps[app['id']] = e
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
newpage.save(txt, summary='Build log')
except:
- print "Error while attempting to publish build log"
+ logging.info("Error while attempting to publish build log")
for app in build_succeeded:
- print "success: %s" % (app['id'])
+ logging.info("success: %s" % (app['id']))
if not options.verbose:
for fa in failed_apps:
- print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
+ logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
- print "Finished."
+ logging.info("Finished.")
if len(build_succeeded) > 0:
- print str(len(build_succeeded)) + ' builds succeeded'
+ logging.info(str(len(build_succeeded)) + ' builds succeeded')
if len(failed_apps) > 0:
- print str(len(failed_apps)) + ' builds failed'
+ logging.info(str(len(failed_apps)) + ' builds failed')
sys.exit(0)
import Queue
import threading
import magic
+import logging
import metadata
if config is not None:
return config
if not os.path.isfile(config_file):
- print "Missing config file - is this a repo directory?"
+ logging.critical("Missing config file - is this a repo directory?")
sys.exit(2)
options = opts
- if not hasattr(options, 'verbose'):
- options.verbose = False
defconfig = {
'sdk_path': "$ANDROID_HOME",
}
config = {}
- if options.verbose:
- print "Reading %s..." % config_file
+ logging.info("Reading %s" % config_file)
execfile(config_file, config)
if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
st = os.stat(config_file)
if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
- print "WARNING: unsafe permissions on {0} (should be 0600)!".format(config_file)
+ logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
for k, v in defconfig.items():
if k not in config:
allids = [app["id"] for app in allapps]
for p in vercodes:
if p not in allids:
- print "No such package: %s" % p
+ logging.critical("No such package: %s" % p)
raise Exception("Found invalid app ids in arguments")
error = False
allvcs = [b['vercode'] for b in app['builds']]
for v in vercodes[app['id']]:
if v not in allvcs:
- print "No such vercode %s for app %s" % (v, app['id'])
+ logging.critical("No such vercode %s for app %s" % (v, app['id']))
if error:
raise Exception("Found invalid vercodes for some apps")
writeback = False
else:
deleterepo = True
- print "*** Repository details changed - deleting ***"
+ logging.info("Repository details changed - deleting")
else:
deleterepo = True
- print "*** Repository details missing - deleting ***"
+ logging.info("Repository details missing - deleting")
if deleterepo:
shutil.rmtree(self.local)
# fdroidserver) and then we'll proceed to destroy it! This is called as
# a safety check.
def checkrepo(self):
- p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
- stdout=subprocess.PIPE, cwd=self.local)
- result = p.communicate()[0].rstrip()
+ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
+ result = p.stdout.rstrip()
if not result.endswith(self.local):
raise VCSException('Repository mismatch')
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
- # Brand new checkout...
+ # Brand new checkout
if subprocess.call(['git', 'clone', self.remote, self.local]) != 0:
raise VCSException("Git clone failed")
self.checkrepo()
else:
self.checkrepo()
- # Discard any working tree changes...
+ # Discard any working tree changes
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
raise VCSException("Git reset failed")
# Remove untracked files now, in case they're tracked in the target
- # revision (it happens!)...
+ # revision (it happens!)
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
raise VCSException("Git clean failed")
if not self.refreshed:
- # Get latest commits and tags from remote...
+ # Get latest commits and tags from remote
if subprocess.call(['git', 'fetch', 'origin'],
cwd=self.local) != 0:
raise VCSException("Git fetch failed")
cwd=self.local) != 0:
raise VCSException("Git fetch failed")
self.refreshed = True
- # Check out the appropriate revision...
+ # Check out the appropriate revision
rev = str(rev if rev else 'origin/master')
if subprocess.call(['git', 'checkout', '-f', rev], cwd=self.local) != 0:
raise VCSException("Git checkout failed")
- # Get rid of any uncontrolled files left behind...
+ # Get rid of any uncontrolled files left behind
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
raise VCSException("Git clean failed")
def initsubmodules(self):
self.checkrepo()
- if subprocess.call(['git', 'submodule', 'init'],
- cwd=self.local) != 0:
- raise VCSException("Git submodule init failed")
- if subprocess.call(['git', 'submodule', 'update'],
- cwd=self.local) != 0:
- raise VCSException("Git submodule update failed")
- if subprocess.call(['git', 'submodule', 'foreach',
+ if subprocess.call(['git', 'submodule', 'foreach', '--recursive',
'git', 'reset', '--hard'],
cwd=self.local) != 0:
raise VCSException("Git submodule reset failed")
- if subprocess.call(['git', 'submodule', 'foreach',
+ if subprocess.call(['git', 'submodule', 'foreach', '--recursive',
'git', 'clean', '-dffx'],
cwd=self.local) != 0:
raise VCSException("Git submodule clean failed")
+ if subprocess.call(['git', 'submodule', 'update',
+ '--init', '--force', '--recursive'],
+ cwd=self.local) != 0:
+ raise VCSException("Git submodule update failed")
def gettags(self):
self.checkrepo()
- p = subprocess.Popen(['git', 'tag'],
- stdout=subprocess.PIPE, cwd=self.local)
- return p.communicate()[0].splitlines()
+ p = FDroidPopen(['git', 'tag'], cwd=self.local)
+ return p.stdout.splitlines()
class vcs_gitsvn(vcs):
# fdroidserver) and then we'll proceed to destory it! This is called as
# a safety check.
def checkrepo(self):
- p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
- stdout=subprocess.PIPE, cwd=self.local)
- result = p.communicate()[0].rstrip()
+ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
+ result = p.stdout.rstrip()
if not result.endswith(self.local):
raise VCSException('Repository mismatch')
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
- # Brand new checkout...
+ # Brand new checkout
gitsvn_cmd = '%sgit svn clone %s' % self.userargs()
if ';' in self.remote:
remote_split = self.remote.split(';')
self.checkrepo()
else:
self.checkrepo()
- # Discard any working tree changes...
+ # Discard any working tree changes
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
raise VCSException("Git reset failed")
# Remove untracked files now, in case they're tracked in the target
- # revision (it happens!)...
+ # revision (it happens!)
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
raise VCSException("Git clean failed")
if not self.refreshed:
- # Get new commits and tags from repo...
+ # Get new commits and tags from repo
if subprocess.call(['%sgit svn rebase %s' % self.userargs()],
cwd=self.local, shell=True) != 0:
raise VCSException("Git svn rebase failed")
if rev:
nospaces_rev = rev.replace(' ', '%20')
# Try finding a svn tag
- p = subprocess.Popen(['git', 'checkout', 'tags/' + nospaces_rev],
- cwd=self.local, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = p.communicate()
- if p.returncode == 0:
- print out
- else:
+ p = FDroidPopen(['git', 'checkout', 'tags/' + nospaces_rev], cwd=self.local)
+ if p.returncode != 0:
# No tag found, normal svn rev translation
# Translate svn rev into git format
rev_split = rev.split('/')
treeish = 'master'
svn_rev = rev
- p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + svn_rev, treeish],
- cwd=self.local, stdout=subprocess.PIPE)
- git_rev = p.communicate()[0].rstrip()
+ p = FDroidPopen(['git', 'svn', 'find-rev', 'r' + svn_rev, treeish],
+ cwd=self.local)
+ git_rev = p.stdout.rstrip()
if p.returncode != 0 or not git_rev:
# Try a plain git checkout as a last resort
- p = subprocess.Popen(['git', 'checkout', rev], cwd=self.local,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = p.communicate()
- if p.returncode == 0:
- print out
- else:
+ p = FDroidPopen(['git', 'checkout', rev], cwd=self.local)
+ if p.returncode != 0:
raise VCSException("No git treeish found and direct git checkout failed")
else:
# Check out the git rev equivalent to the svn rev
- p = subprocess.Popen(['git', 'checkout', git_rev], cwd=self.local,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = p.communicate()
- if p.returncode == 0:
- print out
- else:
+ p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local)
+ if p.returncode != 0:
raise VCSException("Git svn checkout failed")
- # Get rid of any uncontrolled files left behind...
+ # Get rid of any uncontrolled files left behind
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
raise VCSException("Git clean failed")
def getref(self):
self.checkrepo()
- p = subprocess.Popen(['git', 'svn', 'find-rev', 'HEAD'],
- stdout=subprocess.PIPE, cwd=self.local)
- return p.communicate()[0].strip()
+ p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
+ if p.returncode != 0:
+ return None
+ return p.stdout.strip()
class vcs_svn(vcs):
raise VCSException("Svn update failed")
def getref(self):
- p = subprocess.Popen(['svn', 'info'],
- stdout=subprocess.PIPE, cwd=self.local)
+ p = FDroidPopen(['svn', 'info'], cwd=self.local)
for line in p.communicate()[0].splitlines():
if line and line.startswith('Last Changed Rev: '):
return line[18:]
+ return None
class vcs_hg(vcs):
if subprocess.call(['hg', 'update', '-C', rev],
cwd=self.local) != 0:
raise VCSException("Hg checkout failed")
- p = subprocess.Popen(['hg', 'purge', '--all'], stdout=subprocess.PIPE,
- cwd=self.local)
- result = p.communicate()[0]
+ p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local)
# Also delete untracked files, we have to enable purge extension for that:
- if "'purge' is provided by the following extension" in result:
+ if "'purge' is provided by the following extension" in p.stdout:
with open(self.local+"/.hg/hgrc", "a") as myfile:
myfile.write("\n[extensions]\nhgext.purge=\n")
if subprocess.call(['hg', 'purge', '--all'], cwd=self.local) != 0:
raise VCSException("HG purge failed")
def gettags(self):
- p = subprocess.Popen(['hg', 'tags', '-q'],
- stdout=subprocess.PIPE, cwd=self.local)
- return p.communicate()[0].splitlines()[1:]
+ p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local)
+ return p.stdout.splitlines()[1:]
class vcs_bzr(vcs):
raise VCSException("Bzr revert failed")
def gettags(self):
- p = subprocess.Popen(['bzr', 'tags'],
- stdout=subprocess.PIPE, cwd=self.local)
+ p = FDroidPopen(['bzr', 'tags'], cwd=self.local)
return [tag.split(' ')[0].strip() for tag in
- p.communicate()[0].splitlines()]
+ p.stdout.splitlines()]
def retrieve_string(xml_dir, string):
if string.startswith('@string/'):
relpath = os.path.join(root_dir, path)
if not os.path.isdir(relpath):
continue
- if options.verbose:
- print "Found subproject %s..." % path
+ logging.info("Found subproject at %s" % path)
subprojects.append(path)
return subprojects
% name, p.stdout)
if srclib["Update Project"] == "Yes" and not (autoupdate and number):
- print "Updating srclib %s at path %s" % (name, libdir)
+ logging.info("Updating srclib %s at path %s" % (name, libdir))
cmd = [os.path.join(config['sdk_path'], 'tools', 'android'),
'update', 'project', '-p', libdir]
if target:
# 'srclibpaths' is information on the srclibs being used
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
- # Optionally, the actual app source can be in a subdirectory...
+ # Optionally, the actual app source can be in a subdirectory
if 'subdir' in build:
root_dir = os.path.join(build_dir, build['subdir'])
else:
root_dir = build_dir
- # Get a working copy of the right revision...
- print "Getting source for revision " + build['commit']
+ # Get a working copy of the right revision
+ logging.info("Getting source for revision " + build['commit'])
vcs.gotorevision(build['commit'])
# Check that a subdir (if we're using one) exists. This has to happen
- # after the checkout, since it might not exist elsewhere...
+ # after the checkout, since it might not exist elsewhere
if not os.path.exists(root_dir):
raise BuildException('Missing subdir ' + root_dir)
- # Initialise submodules if requred...
+ # Initialise submodules if requred
if build['submodules']:
- if options.verbose:
- print "Initialising submodules..."
+ logging.info("Initialising submodules")
vcs.initsubmodules()
- # Run an init command if one is required...
+ # Run an init command if one is required
if 'init' in build:
cmd = replace_config_vars(build['init'])
- if options.verbose:
- print "Running 'init' commands in %s" % root_dir
+ logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
if 'patch' in build:
for patch in build['patch'].split(';'):
patch = patch.strip()
- print "Applying " + patch
+ logging.info("Applying " + patch)
patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1',
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
- # Get required source libraries...
+ # Get required source libraries
srclibpaths = []
updatemode = build.get('update', 'auto')
if 'srclibs' in build:
target=build['target'] if 'target' in build else None
- print "Collecting source libraries..."
+ logging.info("Collecting source libraries")
for lib in build['srclibs'].split(';'):
srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
target=target, preponly=onserver, autoupdate=(updatemode=='auto')))
update_dirs = ['.'] + ant_subprojects(root_dir)
else:
update_dirs = [d.strip() for d in updatemode.split(';')]
- # Force build.xml update if necessary...
+ # Force build.xml update if necessary
if updatemode == 'force' or 'target' in build:
if updatemode == 'force':
update_dirs = ['.']
buildxml = os.path.join(root_dir, 'build.xml')
if os.path.exists(buildxml):
- print 'Force-removing old build.xml'
+ logging.info('Force-removing old build.xml')
os.remove(buildxml)
for d in update_dirs:
# Clean update dirs via ant
p = FDroidPopen(['ant', 'clean'], cwd=subdir)
dparms = parms + ['-p', d]
- if options.verbose:
- if d == '.':
- print "Updating main project..."
- else:
- print "Updating subproject %s..." % d
+ if d == '.':
+ logging.info("Updating main project")
+ else:
+ logging.info("Updating subproject %s" % d)
p = FDroidPopen(dparms, cwd=root_dir)
# Check to see whether an error was returned without a proper exit
# code (this is the case for the 'no target set or target invalid'
raise BuildException("Failed to update project at %s" % d,
p.stdout)
- # Update the local.properties file...
+ # Update the local.properties file
localprops = [ os.path.join(build_dir, 'local.properties') ]
if 'subdir' in build:
localprops += [ os.path.join(root_dir, 'local.properties') ]
for path in localprops:
if not os.path.isfile(path):
continue
- if options.verbose:
- print "Updating properties file at %s" % path
+ logging.info("Updating properties file at %s" % path)
f = open(path, 'r')
props = f.read()
f.close()
props += '\n'
# Fix old-fashioned 'sdk-location' by copying
- # from sdk.dir, if necessary...
+ # from sdk.dir, if necessary
if build['oldsdkloc']:
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S|re.M).group(1)
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % ['sdk_path']
if 'ndk_path' in config:
- # Add ndk location...
+ # Add ndk location
props += "ndk.dir=%s\n" % config['ndk_path']
props += "ndk-location=%s\n" % config['ndk_path']
- # Add java.encoding if necessary...
+ # Add java.encoding if necessary
if 'encoding' in build:
props += "java.encoding=%s\n" % build['encoding']
f = open(path, 'w')
'build.gradle'], cwd=gradle_dir)
# Remove forced debuggable flags
- print "Removing debuggable flags..."
+ logging.info("Removing debuggable flags")
for path in manifest_paths(root_dir, flavour):
if not os.path.isfile(path):
continue
's/android:debuggable="[^"]*"//g', path]) != 0:
raise BuildException("Failed to remove debuggable flags")
- # Insert version code and number into the manifest if necessary...
+ # Insert version code and number into the manifest if necessary
if build['forceversion']:
- print "Changing the version name..."
+ logging.info("Changing the version name")
for path in manifest_paths(root_dir, flavour):
if not os.path.isfile(path):
continue
path]) != 0:
raise BuildException("Failed to amend build.gradle")
if build['forcevercode']:
- print "Changing the version code..."
+ logging.info("Changing the version code")
for path in manifest_paths(root_dir, flavour):
if not os.path.isfile(path):
continue
path]) != 0:
raise BuildException("Failed to amend build.gradle")
- # Delete unwanted files...
+ # Delete unwanted files
if 'rm' in build:
for part in build['rm'].split(';'):
dest = os.path.join(build_dir, part.strip())
rdest = os.path.abspath(dest)
- if options.verbose:
- print "Removing {0}".format(rdest)
+ logging.info("Removing {0}".format(rdest))
if not rdest.startswith(os.path.abspath(build_dir)):
raise BuildException("rm for {1} is outside build root {0}".format(
os.path.abspath(build_dir),os.path.abspath(dest)))
else:
subprocess.call('rm -rf ' + rdest, shell=True)
else:
- if options.verbose:
- print "...but it didn't exist"
+ logging.info("...but it didn't exist")
- # Fix apostrophes translation files if necessary...
+ # Fix apostrophes translation files if necessary
if build['fixapos']:
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
for filename in files:
os.path.join(root, filename)]) != 0:
raise BuildException("Failed to amend " + filename)
- # Fix translation files if necessary...
+ # Fix translation files if necessary
if build['fixtrans']:
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
for filename in files:
else:
index += 1
# We only want to insert the positional arguments
- # when there is more than one argument...
+ # when there is more than one argument
if oldline != line:
if num > 2:
changed = True
remove_signing_keys(build_dir)
- # Add required external libraries...
+ # Add required external libraries
if 'extlibs' in build:
- print "Collecting prebuilt libraries..."
+ logging.info("Collecting prebuilt libraries")
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
os.mkdir(libsdir)
for lib in build['extlibs'].split(';'):
lib = lib.strip()
- if options.verbose:
- print "...installing extlib {0}".format(lib)
+ logging.info("...installing extlib {0}".format(lib))
libf = os.path.basename(lib)
libsrc = os.path.join(extlib_dir, lib)
if not os.path.exists(libsrc):
raise BuildException("Missing extlib file {0}".format(libsrc))
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
- # Run a pre-build command if one is required...
+ # Run a pre-build command if one is required
if 'prebuild' in build:
cmd = replace_config_vars(build['prebuild'])
- # Substitute source library paths into prebuild commands...
+ # Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
- if options.verbose:
- print "Running 'prebuild' commands in %s" % root_dir
+ logging.info("Running 'prebuild' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
return False
def removeproblem(what, fd, fp):
- print 'Removing %s at %s' % (what, fd)
+ logging.info('Removing %s at %s' % (what, fd))
os.remove(fp)
def handleproblem(what, fd, fp):
problems.append('Found %s at %s' % (what, fd))
def warnproblem(what, fd, fp):
- print 'Warning: Found %s at %s' % (what, fd)
+ logging.info('Warning: Found %s at %s' % (what, fd))
- # Iterate through all files in the source code...
+ # Iterate through all files in the source code
for r,d,f in os.walk(build_dir):
for curfile in f:
if '/.hg' in r or '/.git' in r or '/.svn' in r:
continue
- # Path (relative) to the file...
+ # Path (relative) to the file
fp = os.path.join(r, curfile)
fd = fp[len(build_dir):]
- # Check if this file has been explicitly excluded from scanning...
+ # Check if this file has been explicitly excluded from scanning
if toignore(fd):
continue
ms.close()
# Presence of a jni directory without buildjni=yes might
- # indicate a problem... (if it's not a problem, explicitly use
+ # indicate a problem (if it's not a problem, explicitly use
# buildjni=no to bypass this check)
if (os.path.exists(os.path.join(root_dir, 'jni')) and
thisbuild.get('buildjni') is None):
:param apkfile: full path to the apk to check"""
- p = subprocess.Popen([os.path.join(config['sdk_path'],
+ p = FDroidPopen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
- stdout=subprocess.PIPE)
- output = p.communicate()[0]
+ output=False)
if p.returncode != 0:
- print "ERROR: Failed to get apk manifest information"
+ logging.critical("Failed to get apk manifest information")
sys.exit(1)
- for line in output.splitlines():
+ for line in p.stdout.splitlines():
if 'android:debuggable' in line and not line.endswith('0x0'):
return True
return False
class PopenResult:
returncode = None
stdout = ''
- stderr = ''
- stdout_apk = ''
-def FDroidPopen(commands, cwd=None):
+def FDroidPopen(commands, cwd=None, output=True):
"""
Run a command and capture the output.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
+ :param output: whether to print output as it is fetched
:returns: A PopenResult.
"""
- if options.verbose:
- if cwd:
- print "Directory: %s" % cwd
- print " > %s" % ' '.join(commands)
+ if cwd:
+ logging.info("Directory: %s" % cwd)
+ logging.info("> %s" % ' '.join(commands))
result = PopenResult()
p = subprocess.Popen(commands, cwd=cwd,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ stdin=subprocess.PIPE)
stdout_queue = Queue.Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
while not stdout_reader.eof():
while not stdout_queue.empty():
line = stdout_queue.get()
- if options.verbose:
+ if output and options.verbose:
# Output directly to console
sys.stdout.write(line)
sys.stdout.flush()
else:
o.write(line)
- if changed and options.verbose:
- print "Cleaned build.gradle of keysigning configs at %s" % path
+ if changed:
+ logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
for propfile in ('build.properties', 'default.properties', 'ant.properties'):
if propfile in files:
else:
o.write(line)
- if changed and options.verbose:
- print "Cleaned %s of keysigning configs at %s" % (propfile,path)
+ if changed:
+ logging.info("Cleaned %s of keysigning configs at %s" % (propfile,path))
def replace_config_vars(cmd):
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
~ianmdlvl / fdroidserver.git / commitdiff