import base64
import xml.etree.ElementTree as XMLElementTree
+from binascii import hexlify
+from datetime import datetime
from distutils.version import LooseVersion
from queue import Queue
from zipfile import ZipFile
+from pyasn1.codec.der import decoder, encoder
+from pyasn1_modules import rfc2315
+from pyasn1.error import PyAsn1Error
+
+from distutils.util import strtobool
+
import fdroidserver.metadata
+from fdroidserver.exception import FDroidException, VCSException, BuildException
from .asynchronousfilereader import AsynchronousFileReader
+# A signature block file with a .DSA, .RSA, or .EC extension
+CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
+APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
+STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
+
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
config = None
'r11c': None,
'r12b': "$ANDROID_NDK",
'r13b': None,
+ 'r14b': None,
+ 'r15c': None,
},
'qt_sdk_path': None,
'build_tools': "25.0.2",
'gradle': 'gradle',
'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
+ 'allow_disabled_algorithms': False,
'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Summary': 80,
- 'Description': 4000,
+ 'author': 256,
+ 'name': 30,
+ 'summary': 80,
+ 'description': 4000,
+ 'video': 256,
+ 'whatsNew': 500,
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
with io.open(config_file, "rb") as f:
code = compile(f.read(), config_file, 'exec')
exec(code, None, config)
- elif len(get_local_metadata_files()) == 0:
- logging.critical("Missing config file - is this a repo directory?")
- sys.exit(2)
+ else:
+ logging.debug("No config.py found - using defaults.")
+
+ for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
+ if k in config:
+ if not type(config[k]) in (str, list, tuple):
+ logging.warn('"' + k + '" will be in random order!'
+ + ' Use () or [] brackets if order is important!')
# smartcardoptions must be a list since its command line args for Popen
if 'smartcardoptions' in config:
if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
st = os.stat(config_file)
if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
- logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
+ logging.warning("unsafe permissions on {0} (should be 0600)!".format(config_file))
fill_config_defaults(config)
- for k in ["keystorepass", "keypass"]:
- if k in config:
- write_password_file(k)
-
for k in ["repo_description", "archive_description"]:
if k in config:
config[k] = clean_description(config[k])
def ensure_build_tools_exists(thisconfig):
if not test_sdk_exists(thisconfig):
- sys.exit(3)
+ raise FDroidException("Android SDK not found.")
build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
if not os.path.isdir(versioned_build_tools):
- logging.critical('Android Build Tools path "'
- + versioned_build_tools + '" does not exist!')
- sys.exit(3)
-
-
-def write_password_file(pwtype, password=None):
- '''
- writes out passwords to a protected file instead of passing passwords as
- command line argments
- '''
- filename = '.fdroid.' + pwtype + '.txt'
- fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
- if password is None:
- os.write(fd, config[pwtype].encode('utf-8'))
- else:
- os.write(fd, password.encode('utf-8'))
- os.close(fd)
- config[pwtype + 'file'] = filename
+ raise FDroidException(
+ 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
def get_local_metadata_files():
def read_pkg_args(args, allow_vercodes=False):
"""
- Given the arguments in the form of multiple appid:[vc] strings, this returns
- a dictionary with the set of vercodes specified for each package.
+ :param args: arguments in the form of multiple appid:[vc] strings
+ :returns: a dictionary with the set of vercodes specified for each package
"""
vercodes = {}
return "%s_%s.apk" % (app.id, build.versionCode)
+def get_toolsversion_logname(app, build):
+ return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
+
+
def getsrcname(app, build):
return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
# Derived classes need to implement this. It's called once basic checking
# has been performend.
- def gotorevisionx(self, rev):
+ def gotorevisionx(self, rev): # pylint: disable=unused-argument
raise VCSException("This VCS type doesn't define gotorevisionx")
# Initialise and update submodules
os.path.join(root, 'AndroidManifest.xml'))
-vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
-vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
+vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
+vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
continue
logging.debug("Parsing manifest at {0}".format(path))
- gradle = has_extension(path, 'gradle')
version = None
vercode = None
package = None
- if gradle:
+ if has_extension(path, 'gradle'):
with open(path, 'r') as f:
for line in f:
if gradle_comment.match(line):
if max_version is None and version is not None:
max_version = version
- if max_vercode is None or (vercode is not None and vercode > max_vercode):
+ if vercode is not None \
+ and (max_vercode is None or vercode > max_vercode):
if not ignoresearch or not ignoresearch(version):
if version is not None:
max_version = version
return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
-class FDroidException(Exception):
-
- def __init__(self, value, detail=None):
- self.value = value
- self.detail = detail
-
- def shortened_detail(self):
- if len(self.detail) < 16000:
- return self.detail
- return '[...]\n' + self.detail[-16000:]
-
- def get_wikitext(self):
- ret = repr(self.value) + "\n"
- if self.detail:
- ret += "=detail=\n"
- ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
- return ret
-
- def __str__(self):
- ret = self.value
- if self.detail:
- ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
- return ret
-
-
-class VCSException(FDroidException):
- pass
-
-
-class BuildException(FDroidException):
- pass
-
-
# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
-# Prepare the source code for a particular build
-# 'vcs' - the appropriate vcs object for the application
-# 'app' - the application details from the metadata
-# 'build' - the build details from the metadata
-# 'build_dir' - the path to the build directory, usually
-# 'build/app.id'
-# 'srclib_dir' - the path to the source libraries directory, usually
-# 'build/srclib'
-# 'extlib_dir' - the path to the external libraries directory, usually
-# 'build/extlib'
-# Returns the (root, srclibpaths) where:
-# 'root' is the root directory, which may be the same as 'build_dir' or may
-# be a subdirectory of it.
-# 'srclibpaths' is information on the srclibs being used
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
+ """ Prepare the source code for a particular build
+
+ :param vcs: the appropriate vcs object for the application
+ :param app: the application details from the metadata
+ :param build: the build details from the metadata
+ :param build_dir: the path to the build directory, usually 'build/app.id'
+ :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
+ :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
+
+ Returns the (root, srclibpaths) where:
+ :param root: is the root directory, which may be the same as 'build_dir' or may
+ be a subdirectory of it.
+ :param srclibpaths: is information on the srclibs being used
+ """
# Optionally, the actual app source can be in a subdirectory
if build.subdir:
class KnownApks:
+ """permanent store of existing APKs with the date they were added
+
+ This is currently the only way to permanently store the "updated"
+ date of APKs.
+ """
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
if len(t) == 2:
self.apks[t[0]] = (t[1], None)
else:
- self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
+ self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
self.changed = False
def writeifchanged(self):
appid, added = app
line = apk + ' ' + appid
if added:
- line += ' ' + time.strftime('%Y-%m-%d', added)
+ line += ' ' + added.strftime('%Y-%m-%d')
lst.append(line)
with open(self.path, 'w', encoding='utf8') as f:
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
- # Record an apk (if it's new, otherwise does nothing)
- # Returns the date it was added.
- def recordapk(self, apk, app, default_date=None):
- if apk not in self.apks:
+ def recordapk(self, apkName, app, default_date=None):
+ '''
+ Record an apk (if it's new, otherwise does nothing)
+ Returns the date it was added as a datetime instance
+ '''
+ if apkName not in self.apks:
if default_date is None:
- default_date = time.gmtime(time.time())
- self.apks[apk] = (app, default_date)
+ default_date = datetime.utcnow()
+ self.apks[apkName] = (app, default_date)
self.changed = True
- _, added = self.apks[apk]
+ _, added = self.apks[apkName]
return added
# Look up information - given the 'apkname', returns (app id, date added/None).
def get_file_extension(filename):
"""get the normalized file extension, can be blank string but never None"""
-
+ if isinstance(filename, bytes):
+ filename = filename.decode('utf-8')
return os.path.splitext(filename)[1].lower()[1:]
-def isApkAndDebuggable(apkfile, config):
+def get_apk_debuggable_aapt(apkfile):
+ p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
+ output=False)
+ if p.returncode != 0:
+ raise FDroidException("Failed to get apk manifest information")
+ for line in p.output.splitlines():
+ if 'android:debuggable' in line and not line.endswith('0x0'):
+ return True
+ return False
+
+
+def get_apk_debuggable_androguard(apkfile):
+ try:
+ from androguard.core.bytecodes.apk import APK
+ except ImportError:
+ raise FDroidException("androguard library is not installed and aapt not present")
+
+ apkobject = APK(apkfile)
+ if apkobject.is_valid_APK():
+ debuggable = apkobject.get_element("application", "debuggable")
+ if debuggable is not None:
+ return bool(strtobool(debuggable))
+ return False
+
+
+def isApkAndDebuggable(apkfile):
"""Returns True if the given file is an APK and is debuggable
:param apkfile: full path to the apk to check"""
if get_file_extension(apkfile) != 'apk':
return False
- p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
- output=False)
- if p.returncode != 0:
- logging.critical("Failed to get apk manifest information")
- sys.exit(1)
+ if SdkToolsPopen(['aapt', 'version'], output=False):
+ return get_apk_debuggable_aapt(apkfile)
+ else:
+ return get_apk_debuggable_androguard(apkfile)
+
+
+def get_apk_id_aapt(apkfile):
+ """Extrat identification information from APK using aapt.
+
+ :param apkfile: path to an APK file.
+ :returns: triplet (appid, version code, version name)
+ """
+ r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
+ p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
for line in p.output.splitlines():
- if 'android:debuggable' in line and not line.endswith('0x0'):
- return True
- return False
+ m = r.match(line)
+ if m:
+ return m.group('appid'), m.group('vercode'), m.group('vername')
+ raise FDroidException("reading identification failed, APK invalid: '{}'".format(apkfile))
class PopenResult:
config[cmd] = find_sdk_tools_cmd(commands[0])
abscmd = config[cmd]
if abscmd is None:
- logging.critical("Could not find '%s' on your system" % cmd)
- sys.exit(1)
+ raise FDroidException("Could not find '%s' on your system" % cmd)
if cmd == 'aapt':
test_aapt_version(config['aapt'])
return FDroidPopen([abscmd] + commands[1:],
cwd=cwd, output=output)
-def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
"""
Run a command and capture the possibly huge output as bytes.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
+ :param envs: a optional dictionary of environment variables and their values
:returns: A PopenResult.
"""
if env is None:
set_FDroidPopen_env()
+ process_env = env.copy()
+ if envs is not None and len(envs) > 0:
+ process_env.update(envs)
+
if cwd:
cwd = os.path.normpath(cwd)
logging.debug("Directory: %s" % cwd)
result = PopenResult()
p = None
try:
- p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
+ p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
stdout=subprocess.PIPE, stderr=stderr_param)
except OSError as e:
raise BuildException("OSError while trying to execute " +
result.returncode = p.wait()
result.output = buf.getvalue()
buf.close()
+ # make sure all filestreams of the subprocess are closed
+ for streamvar in ['stdin', 'stdout', 'stderr']:
+ if hasattr(p, streamvar):
+ stream = getattr(p, streamvar)
+ if stream:
+ stream.close()
return result
-def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
"""
Run a command and capture the possibly huge output as a str.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
+ :param envs: a optional dictionary of environment variables and their values
:returns: A PopenResult.
"""
- result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
- result.output = result.output.decode('utf-8')
+ result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
+ result.output = result.output.decode('utf-8', 'ignore')
return result
env[n] = build.ndk_path()
+def replace_build_vars(cmd, build):
+ cmd = cmd.replace('$$COMMIT$$', build.commit)
+ cmd = cmd.replace('$$VERSION$$', build.versionName)
+ cmd = cmd.replace('$$VERCODE$$', build.versionCode)
+ return cmd
+
+
def replace_config_vars(cmd, build):
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
cmd = cmd.replace('$$NDK$$', build.ndk_path())
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
if build is not None:
- cmd = cmd.replace('$$COMMIT$$', build.commit)
- cmd = cmd.replace('$$VERSION$$', build.versionName)
- cmd = cmd.replace('$$VERCODE$$', build.versionCode)
+ cmd = replace_build_vars(cmd, build)
return cmd
apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
+def metadata_get_sigdir(appid, vercode=None):
+ """Get signature directory for app"""
+ if vercode:
+ return os.path.join('metadata', appid, 'signatures', vercode)
+ else:
+ return os.path.join('metadata', appid, 'signatures')
+
+
+def apk_extract_signatures(apkpath, outdir, manifest=True):
+ """Extracts a signature files from APK and puts them into target directory.
+
+ :param apkpath: location of the apk
+ :param outdir: folder where the extracted signature files will be stored
+ :param manifest: (optionally) disable extracting manifest file
+ """
+ with ZipFile(apkpath, 'r') as in_apk:
+ for f in in_apk.infolist():
+ if apk_sigfile.match(f.filename) or \
+ (manifest and f.filename == 'META-INF/MANIFEST.MF'):
+ newpath = os.path.join(outdir, os.path.basename(f.filename))
+ with open(newpath, 'wb') as out_file:
+ out_file.write(in_apk.read(f.filename))
+
+
def verify_apks(signed_apk, unsigned_apk, tmp_dir):
"""Verify that two apks are the same
One of the inputs is signed, the other is unsigned. The signature metadata
is transferred from the signed to the unsigned apk, and then jarsigner is
used to verify that the signature from the signed apk is also varlid for
- the unsigned one.
+ the unsigned one. If the APK given as unsigned actually does have a
+ signature, it will be stripped out and ignored.
+
+ There are two SHA1 git commit IDs that fdroidserver includes in the builds
+ it makes: fdroidserverid and buildserverid. Originally, these were inserted
+ into AndroidManifest.xml, but that makes the build not reproducible. So
+ instead they are included as separate files in the APK's META-INF/ folder.
+ If those files exist in the signed APK, they will be part of the signature
+ and need to also be included in the unsigned APK for it to validate.
+
:param signed_apk: Path to a signed apk file
:param unsigned_apk: Path to an unsigned apk file expected to match it
:param tmp_dir: Path to directory for temporary files
:returns: None if the verification is successful, otherwise a string
describing what went wrong.
"""
- with ZipFile(signed_apk) as signed_apk_as_zip:
- meta_inf_files = ['META-INF/MANIFEST.MF']
- for f in signed_apk_as_zip.namelist():
- if apk_sigfile.match(f):
- meta_inf_files.append(f)
- if len(meta_inf_files) < 3:
- return "Signature files missing from {0}".format(signed_apk)
- signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
- with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
- for meta_inf_file in meta_inf_files:
- unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
-
- if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
- logging.info("...NOT verified - {0}".format(signed_apk))
- return compare_apks(signed_apk, unsigned_apk, tmp_dir)
+
+ signed = ZipFile(signed_apk, 'r')
+ meta_inf_files = ['META-INF/MANIFEST.MF']
+ for f in signed.namelist():
+ if apk_sigfile.match(f) \
+ or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
+ meta_inf_files.append(f)
+ if len(meta_inf_files) < 3:
+ return "Signature files missing from {0}".format(signed_apk)
+
+ tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
+ unsigned = ZipFile(unsigned_apk, 'r')
+ # only read the signature from the signed APK, everything else from unsigned
+ with ZipFile(tmp_apk, 'w') as tmp:
+ for filename in meta_inf_files:
+ tmp.writestr(signed.getinfo(filename), signed.read(filename))
+ for info in unsigned.infolist():
+ if info.filename in meta_inf_files:
+ logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
+ continue
+ if info.filename in tmp.namelist():
+ return "duplicate filename found: " + info.filename
+ tmp.writestr(info, unsigned.read(info.filename))
+ unsigned.close()
+ signed.close()
+
+ verified = verify_apk_signature(tmp_apk)
+
+ if not verified:
+ logging.info("...NOT verified - {0}".format(tmp_apk))
+ return compare_apks(signed_apk, tmp_apk, tmp_dir,
+ os.path.dirname(unsigned_apk))
+
logging.info("...successfully verified")
return None
+def verify_apk_signature(apk, jar=False):
+ """verify the signature on an APK
+
+ Try to use apksigner whenever possible since jarsigner is very
+ shitty: unsigned APKs pass as "verified"! So this has to turn on
+ -strict then check for result 4.
+
+ You can set :param: jar to True if you want to use this method
+ to verify jar signatures.
+ """
+ if set_command_in_config('apksigner'):
+ args = [config['apksigner'], 'verify']
+ if jar:
+ args += ['--min-sdk-version=1']
+ return subprocess.call(args + [apk]) == 0
+ else:
+ logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
+ return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
+
+
+def verify_old_apk_signature(apk):
+ """verify the signature on an archived APK, supporting deprecated algorithms
+
+ F-Droid aims to keep every single binary that it ever published. Therefore,
+ it needs to be able to verify APK signatures that include deprecated/removed
+ algorithms. For example, jarsigner treats an MD5 signature as unsigned.
+
+ jarsigner passes unsigned APKs as "verified"! So this has to turn
+ on -strict then check for result 4.
+
+ """
+
+ _java_security = os.path.join(os.getcwd(), '.java.security')
+ with open(_java_security, 'w') as fp:
+ fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
+
+ return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
+ '-strict', '-verify', apk]) == 4
+
+
apk_badchars = re.compile('''[/ :;'"]''')
-def compare_apks(apk1, apk2, tmp_dir):
+def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
"""Compare two apks
Returns None if the apk content is the same (apart from the signing key),
trying to do the comparison.
"""
+ if not log_dir:
+ log_dir = tmp_dir
+
+ absapk1 = os.path.abspath(apk1)
+ absapk2 = os.path.abspath(apk2)
+
+ if set_command_in_config('diffoscope'):
+ logfilename = os.path.join(log_dir, os.path.basename(absapk1))
+ htmlfile = logfilename + '.diffoscope.html'
+ textfile = logfilename + '.diffoscope.txt'
+ if subprocess.call([config['diffoscope'],
+ '--max-report-size', '12345678', '--max-diff-block-lines', '100',
+ '--html', htmlfile, '--text', textfile,
+ absapk1, absapk2]) != 0:
+ return("Failed to unpack " + apk1)
+
apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
for d in [apk1dir, apk2dir]:
cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
return("Failed to unpack " + apk2)
- # try to find apktool in the path, if it hasn't been manually configed
- if 'apktool' not in config:
- tmp = find_command('apktool')
- if tmp is not None:
- config['apktool'] = tmp
- if 'apktool' in config:
+ if set_command_in_config('apktool'):
if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
cwd=apk1dir) != 0:
return("Failed to unpack " + apk1)
p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
lines = p.output.splitlines()
if len(lines) != 1 or 'META-INF' not in lines[0]:
- meld = find_command('meld')
- if meld is not None:
- p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
+ if set_command_in_config('meld'):
+ p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
return("Unexpected diff output - " + p.output)
# since everything verifies, delete the comparison to keep cruft down
return None
+def set_command_in_config(command):
+ '''Try to find specified command in the path, if it hasn't been
+ manually set in config.py. If found, it is added to the config
+ dict. The return value says whether the command is available.
+
+ '''
+ if command in config:
+ return True
+ else:
+ tmp = find_command(command)
+ if tmp is not None:
+ config[command] = tmp
+ return True
+ return False
+
+
def find_command(command):
'''find the full path of a command, or None if it can't be found in the PATH'''
def genkeystore(localconfig):
- '''Generate a new key with random passwords and add it to new keystore'''
+ """
+ Generate a new key with password provided in :param localconfig and add it to new keystore
+ :return: hexed public key, public key fingerprint
+ """
logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
keystoredir = os.path.dirname(localconfig['keystore'])
if keystoredir is None or keystoredir == '':
if not os.path.exists(keystoredir):
os.makedirs(keystoredir, mode=0o700)
- write_password_file("keystorepass", localconfig['keystorepass'])
- write_password_file("keypass", localconfig['keypass'])
+ env_vars = {
+ 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
+ 'FDROID_KEY_PASS': localconfig['keypass'],
+ }
p = FDroidPopen([config['keytool'], '-genkey',
'-keystore', localconfig['keystore'],
'-alias', localconfig['repo_keyalias'],
'-keyalg', 'RSA', '-keysize', '4096',
'-sigalg', 'SHA256withRSA',
'-validity', '10000',
- '-storepass:file', config['keystorepassfile'],
- '-keypass:file', config['keypassfile'],
- '-dname', localconfig['keydname']])
- # TODO keypass should be sent via stdin
+ '-storepass:env', 'FDROID_KEY_STORE_PASS',
+ '-keypass:env', 'FDROID_KEY_PASS',
+ '-dname', localconfig['keydname']], envs=env_vars)
if p.returncode != 0:
raise BuildException("Failed to generate key", p.output)
os.chmod(localconfig['keystore'], 0o0600)
- # now show the lovely key that was just generated
- p = FDroidPopen([config['keytool'], '-list', '-v',
- '-keystore', localconfig['keystore'],
- '-alias', localconfig['repo_keyalias'],
- '-storepass:file', config['keystorepassfile']])
- logging.info(p.output.strip() + '\n\n')
+ if not options.quiet:
+ # now show the lovely key that was just generated
+ p = FDroidPopen([config['keytool'], '-list', '-v',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
+ logging.info(p.output.strip() + '\n\n')
+ # get the public key
+ p = FDroidPopenBytes([config['keytool'], '-exportcert',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:env', 'FDROID_KEY_STORE_PASS']
+ + config['smartcardoptions'],
+ envs=env_vars, output=False, stderr_to_stdout=False)
+ if p.returncode != 0 or len(p.output) < 20:
+ raise BuildException("Failed to get public key", p.output)
+ pubkey = p.output
+ fingerprint = get_cert_fingerprint(pubkey)
+ return hexlify(pubkey), fingerprint
+
+
+def get_cert_fingerprint(pubkey):
+ """
+ Generate a certificate fingerprint the same way keytool does it
+ (but with slightly different formatting)
+ """
+ digest = hashlib.sha256(pubkey).digest()
+ ret = [' '.join("%02X" % b for b in bytearray(digest))]
+ return " ".join(ret)
-def write_to_config(thisconfig, key, value=None):
- '''write a key/value to the local config.py'''
+def get_certificate(certificate_file):
+ """
+ Extracts a certificate from the given file.
+ :param certificate_file: file bytes (as string) representing the certificate
+ :return: A binary representation of the certificate's public key, or None in case of error
+ """
+ content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
+ if content.getComponentByName('contentType') != rfc2315.signedData:
+ return None
+ content = decoder.decode(content.getComponentByName('content'),
+ asn1Spec=rfc2315.SignedData())[0]
+ try:
+ certificates = content.getComponentByName('certificates')
+ cert = certificates[0].getComponentByName('certificate')
+ except PyAsn1Error:
+ logging.error("Certificates not found.")
+ return None
+ return encoder.encode(cert)
+
+
+def write_to_config(thisconfig, key, value=None, config_file=None):
+ '''write a key/value to the local config.py
+
+ NOTE: only supports writing string variables.
+
+ :param thisconfig: config dictionary
+ :param key: variable name in config.py to be overwritten/added
+ :param value: optional value to be written, instead of fetched
+ from 'thisconfig' dictionary.
+ '''
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
- with open('config.py', 'r', encoding='utf8') as f:
- data = f.read()
- pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
- repl = '\n' + key + ' = "' + value + '"'
- data = re.sub(pattern, repl, data)
- # if this key is not in the file, append it
- if not re.match('\s*' + key + '\s*=\s*"', data):
- data += repl
+ cfg = config_file if config_file else 'config.py'
+
+ # load config file, create one if it doesn't exist
+ if not os.path.exists(cfg):
+ os.mknod(cfg)
+ logging.info("Creating empty " + cfg)
+ with open(cfg, 'r', encoding="utf-8") as f:
+ lines = f.readlines()
+
# make sure the file ends with a carraige return
- if not re.match('\n$', data):
- data += '\n'
- with open('config.py', 'w', encoding='utf8') as f:
- f.writelines(data)
+ if len(lines) > 0:
+ if not lines[-1].endswith('\n'):
+ lines[-1] += '\n'
+
+ # regex for finding and replacing python string variable
+ # definitions/initializations
+ pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
+ repl = key + ' = "' + value + '"'
+ pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
+ repl2 = key + " = '" + value + "'"
+
+ # If we replaced this line once, we make sure won't be a
+ # second instance of this line for this key in the document.
+ didRepl = False
+ # edit config file
+ with open(cfg, 'w', encoding="utf-8") as f:
+ for line in lines:
+ if pattern.match(line) or pattern2.match(line):
+ if not didRepl:
+ line = pattern.sub(repl, line)
+ line = pattern2.sub(repl2, line)
+ f.write(line)
+ didRepl = True
+ else:
+ f.write(line)
+ if not didRepl:
+ f.write('\n')
+ f.write(repl)
+ f.write('\n')
def parse_xml(path):
def is_repo_file(filename):
'''Whether the file in a repo is a build product to be delivered to users'''
+ if isinstance(filename, str):
+ filename = filename.encode('utf-8', errors="surrogateescape")
return os.path.isfile(filename) \
+ and not filename.endswith(b'.asc') \
+ and not filename.endswith(b'.sig') \
and os.path.basename(filename) not in [
- 'index.jar',
- 'index.xml',
- 'index.html',
- 'categories.txt',
+ b'index.jar',
+ b'index_unsigned.jar',
+ b'index.xml',
+ b'index.html',
+ b'index-v1.jar',
+ b'index-v1.json',
+ b'categories.txt',
]
~ianmdlvl / fdroidserver.git / blobdiff