import hashlib
import socket
import base64
-import zipfile
import xml.etree.ElementTree as XMLElementTree
+from binascii import hexlify
from datetime import datetime
from distutils.version import LooseVersion
from queue import Queue
from zipfile import ZipFile
+from pyasn1.codec.der import decoder, encoder
+from pyasn1_modules import rfc2315
+from pyasn1.error import PyAsn1Error
+
+from distutils.util import strtobool
+
import fdroidserver.metadata
+from fdroidserver.exception import FDroidException, VCSException, BuildException
from .asynchronousfilereader import AsynchronousFileReader
+# A signature block file with a .DSA, .RSA, or .EC extension
+CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
+APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
+STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
+
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
config = None
'r11c': None,
'r12b': "$ANDROID_NDK",
'r13b': None,
- 'r14': None,
+ 'r14b': None,
+ 'r15b': None,
},
'qt_sdk_path': None,
'build_tools': "25.0.2",
'gradle': 'gradle',
'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
+ 'allow_disabled_algorithms': False,
'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Summary': 80,
- 'Description': 4000,
+ 'author': 256,
+ 'name': 30,
+ 'summary': 80,
+ 'description': 4000,
+ 'video': 256,
+ 'whatsNew': 500,
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
code = compile(f.read(), config_file, 'exec')
exec(code, None, config)
elif len(get_local_metadata_files()) == 0:
- logging.critical("Missing config file - is this a repo directory?")
- sys.exit(2)
+ raise FDroidException("Missing config file - is this a repo directory?")
for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
if k in config:
fill_config_defaults(config)
- for k in ["keystorepass", "keypass"]:
- if k in config:
- write_password_file(k)
-
for k in ["repo_description", "archive_description"]:
if k in config:
config[k] = clean_description(config[k])
def ensure_build_tools_exists(thisconfig):
if not test_sdk_exists(thisconfig):
- sys.exit(3)
+ raise FDroidException("Android SDK not found.")
build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
if not os.path.isdir(versioned_build_tools):
- logging.critical('Android Build Tools path "'
- + versioned_build_tools + '" does not exist!')
- sys.exit(3)
-
-
-def write_password_file(pwtype, password=None):
- '''
- writes out passwords to a protected file instead of passing passwords as
- command line argments
- '''
- filename = '.fdroid.' + pwtype + '.txt'
- fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
- if password is None:
- os.write(fd, config[pwtype].encode('utf-8'))
- else:
- os.write(fd, password.encode('utf-8'))
- os.close(fd)
- config[pwtype + 'file'] = filename
-
-
-def signjar(jar):
- '''
- sign a JAR file with Java's jarsigner.
-
- This does use old hashing algorithms, i.e. SHA1, but that's not
- broken yet for file verification. This could be set to SHA256,
- but then Android < 4.3 would not be able to verify it.
- https://code.google.com/p/android/issues/detail?id=38321
- '''
- args = [config['jarsigner'], '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile'],
- '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
- jar, config['repo_keyalias']]
- if config['keystore'] == 'NONE':
- args += config['smartcardoptions']
- else: # smardcards never use -keypass
- args += ['-keypass:file', config['keypassfile']]
- p = FDroidPopen(args)
- if p.returncode != 0:
- logging.critical("Failed to sign %s!" % jar)
- sys.exit(1)
-
-
-def sign_index_v1(repodir, json_name):
- """
- sign index-v1.json to make index-v1.jar
-
- This is a bit different than index.jar: instead of their being index.xml
- and index_unsigned.jar, the presense of index-v1.json means that there is
- unsigned data. That file is then stuck into a jar and signed by the
- signing process. index-v1.json is never published to the repo. It is
- included in the binary transparency log, if that is enabled.
- """
- name, ext = get_extension(json_name)
- index_file = os.path.join(repodir, json_name)
- jar_file = os.path.join(repodir, name + '.jar')
- with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
- jar.write(index_file, json_name)
- signjar(jar_file)
+ raise FDroidException(
+ 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
def get_local_metadata_files():
def read_pkg_args(args, allow_vercodes=False):
"""
- Given the arguments in the form of multiple appid:[vc] strings, this returns
- a dictionary with the set of vercodes specified for each package.
+ :param args: arguments in the form of multiple appid:[vc] strings
+ :returns: a dictionary with the set of vercodes specified for each package
"""
vercodes = {}
return "%s_%s.apk" % (app.id, build.versionCode)
+def get_toolsversion_logname(app, build):
+ return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
+
+
def getsrcname(app, build):
return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
# Derived classes need to implement this. It's called once basic checking
# has been performend.
- def gotorevisionx(self, rev):
+ def gotorevisionx(self, rev): # pylint: disable=unused-argument
raise VCSException("This VCS type doesn't define gotorevisionx")
# Initialise and update submodules
return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
-class FDroidException(Exception):
-
- def __init__(self, value, detail=None):
- self.value = value
- self.detail = detail
-
- def shortened_detail(self):
- if len(self.detail) < 16000:
- return self.detail
- return '[...]\n' + self.detail[-16000:]
-
- def get_wikitext(self):
- ret = repr(self.value) + "\n"
- if self.detail:
- ret += "=detail=\n"
- ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
- return ret
-
- def __str__(self):
- ret = self.value
- if self.detail:
- ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
- return ret
-
-
-class VCSException(FDroidException):
- pass
-
-
-class BuildException(FDroidException):
- pass
-
-
# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
class KnownApks:
+ """permanent store of existing APKs with the date they were added
+
+ This is currently the only way to permanently store the "updated"
+ date of APKs.
+ """
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
- def recordapk(self, apk, app, default_date=None):
+ def recordapk(self, apkName, app, default_date=None):
'''
Record an apk (if it's new, otherwise does nothing)
Returns the date it was added as a datetime instance
'''
- if apk not in self.apks:
+ if apkName not in self.apks:
if default_date is None:
default_date = datetime.utcnow()
- self.apks[apk] = (app, default_date)
+ self.apks[apkName] = (app, default_date)
self.changed = True
- _, added = self.apks[apk]
+ _, added = self.apks[apkName]
return added
# Look up information - given the 'apkname', returns (app id, date added/None).
def get_file_extension(filename):
"""get the normalized file extension, can be blank string but never None"""
-
+ if isinstance(filename, bytes):
+ filename = filename.decode('utf-8')
return os.path.splitext(filename)[1].lower()[1:]
-def isApkAndDebuggable(apkfile, config):
- """Returns True if the given file is an APK and is debuggable
-
- :param apkfile: full path to the apk to check"""
-
- if get_file_extension(apkfile) != 'apk':
- return False
-
+def get_apk_debuggable_aapt(apkfile):
p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
output=False)
if p.returncode != 0:
- logging.critical("Failed to get apk manifest information")
- sys.exit(1)
+ raise FDroidException("Failed to get apk manifest information")
for line in p.output.splitlines():
if 'android:debuggable' in line and not line.endswith('0x0'):
return True
return False
+def get_apk_debuggable_androguard(apkfile):
+ try:
+ from androguard.core.bytecodes.apk import APK
+ except ImportError:
+ raise FDroidException("androguard library is not installed and aapt not present")
+
+ apkobject = APK(apkfile)
+ if apkobject.is_valid_APK():
+ debuggable = apkobject.get_element("application", "debuggable")
+ if debuggable is not None:
+ return bool(strtobool(debuggable))
+ return False
+
+
+def isApkAndDebuggable(apkfile):
+ """Returns True if the given file is an APK and is debuggable
+
+ :param apkfile: full path to the apk to check"""
+
+ if get_file_extension(apkfile) != 'apk':
+ return False
+
+ if SdkToolsPopen(['aapt', 'version'], output=False):
+ return get_apk_debuggable_aapt(apkfile)
+ else:
+ return get_apk_debuggable_androguard(apkfile)
+
+
class PopenResult:
def __init__(self):
self.returncode = None
config[cmd] = find_sdk_tools_cmd(commands[0])
abscmd = config[cmd]
if abscmd is None:
- logging.critical("Could not find '%s' on your system" % cmd)
- sys.exit(1)
+ raise FDroidException("Could not find '%s' on your system" % cmd)
if cmd == 'aapt':
test_aapt_version(config['aapt'])
return FDroidPopen([abscmd] + commands[1:],
cwd=cwd, output=output)
-def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
"""
Run a command and capture the possibly huge output as bytes.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
+ :param envs: a optional dictionary of environment variables and their values
:returns: A PopenResult.
"""
if env is None:
set_FDroidPopen_env()
+ process_env = env.copy()
+ if envs is not None and len(envs) > 0:
+ process_env.update(envs)
+
if cwd:
cwd = os.path.normpath(cwd)
logging.debug("Directory: %s" % cwd)
result = PopenResult()
p = None
try:
- p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
+ p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
stdout=subprocess.PIPE, stderr=stderr_param)
except OSError as e:
raise BuildException("OSError while trying to execute " +
return result
-def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
"""
Run a command and capture the possibly huge output as a str.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
+ :param envs: a optional dictionary of environment variables and their values
:returns: A PopenResult.
"""
- result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
+ result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
result.output = result.output.decode('utf-8', 'ignore')
return result
if not verified:
logging.info("...NOT verified - {0}".format(tmp_apk))
- return compare_apks(signed_apk, tmp_apk, tmp_dir, os.path.dirname(unsigned_apk))
+ return compare_apks(signed_apk, tmp_apk, tmp_dir,
+ os.path.dirname(unsigned_apk))
logging.info("...successfully verified")
return None
-def verify_apk_signature(apk):
+def verify_apk_signature(apk, jar=False):
"""verify the signature on an APK
Try to use apksigner whenever possible since jarsigner is very
shitty: unsigned APKs pass as "verified"! So this has to turn on
-strict then check for result 4.
+ You can set :param: jar to True if you want to use this method
+ to verify jar signatures.
"""
if set_command_in_config('apksigner'):
- return subprocess.call([config['apksigner'], 'verify', apk]) == 0
+ args = [config['apksigner'], 'verify']
+ if jar:
+ args += ['--min-sdk-version=1']
+ return subprocess.call(args + [apk]) == 0
else:
logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
+def verify_old_apk_signature(apk):
+ """verify the signature on an archived APK, supporting deprecated algorithms
+
+ F-Droid aims to keep every single binary that it ever published. Therefore,
+ it needs to be able to verify APK signatures that include deprecated/removed
+ algorithms. For example, jarsigner treats an MD5 signature as unsigned.
+
+ jarsigner passes unsigned APKs as "verified"! So this has to turn
+ on -strict then check for result 4.
+
+ """
+
+ _java_security = os.path.join(os.getcwd(), '.java.security')
+ with open(_java_security, 'w') as fp:
+ fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
+
+ return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
+ '-strict', '-verify', apk]) == 4
+
+
apk_badchars = re.compile('''[/ :;'"]''')
p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
lines = p.output.splitlines()
if len(lines) != 1 or 'META-INF' not in lines[0]:
- meld = find_command('meld')
- if meld is not None:
- p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
+ if set_command_in_config('meld'):
+ p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
return("Unexpected diff output - " + p.output)
# since everything verifies, delete the comparison to keep cruft down
def genkeystore(localconfig):
- '''Generate a new key with random passwords and add it to new keystore'''
+ """
+ Generate a new key with password provided in :param localconfig and add it to new keystore
+ :return: hexed public key, public key fingerprint
+ """
logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
keystoredir = os.path.dirname(localconfig['keystore'])
if keystoredir is None or keystoredir == '':
if not os.path.exists(keystoredir):
os.makedirs(keystoredir, mode=0o700)
- write_password_file("keystorepass", localconfig['keystorepass'])
- write_password_file("keypass", localconfig['keypass'])
+ env_vars = {
+ 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
+ 'FDROID_KEY_PASS': localconfig['keypass'],
+ }
p = FDroidPopen([config['keytool'], '-genkey',
'-keystore', localconfig['keystore'],
'-alias', localconfig['repo_keyalias'],
'-keyalg', 'RSA', '-keysize', '4096',
'-sigalg', 'SHA256withRSA',
'-validity', '10000',
- '-storepass:file', config['keystorepassfile'],
- '-keypass:file', config['keypassfile'],
- '-dname', localconfig['keydname']])
- # TODO keypass should be sent via stdin
+ '-storepass:env', 'FDROID_KEY_STORE_PASS',
+ '-keypass:env', 'FDROID_KEY_PASS',
+ '-dname', localconfig['keydname']], envs=env_vars)
if p.returncode != 0:
raise BuildException("Failed to generate key", p.output)
os.chmod(localconfig['keystore'], 0o0600)
- # now show the lovely key that was just generated
- p = FDroidPopen([config['keytool'], '-list', '-v',
- '-keystore', localconfig['keystore'],
- '-alias', localconfig['repo_keyalias'],
- '-storepass:file', config['keystorepassfile']])
- logging.info(p.output.strip() + '\n\n')
+ if not options.quiet:
+ # now show the lovely key that was just generated
+ p = FDroidPopen([config['keytool'], '-list', '-v',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
+ logging.info(p.output.strip() + '\n\n')
+ # get the public key
+ p = FDroidPopenBytes([config['keytool'], '-exportcert',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:env', 'FDROID_KEY_STORE_PASS']
+ + config['smartcardoptions'],
+ envs=env_vars, output=False, stderr_to_stdout=False)
+ if p.returncode != 0 or len(p.output) < 20:
+ raise BuildException("Failed to get public key", p.output)
+ pubkey = p.output
+ fingerprint = get_cert_fingerprint(pubkey)
+ return hexlify(pubkey), fingerprint
+
+
+def get_cert_fingerprint(pubkey):
+ """
+ Generate a certificate fingerprint the same way keytool does it
+ (but with slightly different formatting)
+ """
+ digest = hashlib.sha256(pubkey).digest()
+ ret = [' '.join("%02X" % b for b in bytearray(digest))]
+ return " ".join(ret)
+
+
+def get_certificate(certificate_file):
+ """
+ Extracts a certificate from the given file.
+ :param certificate_file: file bytes (as string) representing the certificate
+ :return: A binary representation of the certificate's public key, or None in case of error
+ """
+ content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
+ if content.getComponentByName('contentType') != rfc2315.signedData:
+ return None
+ content = decoder.decode(content.getComponentByName('content'),
+ asn1Spec=rfc2315.SignedData())[0]
+ try:
+ certificates = content.getComponentByName('certificates')
+ cert = certificates[0].getComponentByName('certificate')
+ except PyAsn1Error:
+ logging.error("Certificates not found.")
+ return None
+ return encoder.encode(cert)
-def write_to_config(thisconfig, key, value=None):
- '''write a key/value to the local config.py'''
+def write_to_config(thisconfig, key, value=None, config_file=None):
+ '''write a key/value to the local config.py
+
+ NOTE: only supports writing string variables.
+
+ :param thisconfig: config dictionary
+ :param key: variable name in config.py to be overwritten/added
+ :param value: optional value to be written, instead of fetched
+ from 'thisconfig' dictionary.
+ '''
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
- with open('config.py', 'r', encoding='utf8') as f:
- data = f.read()
- pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
- repl = '\n' + key + ' = "' + value + '"'
- data = re.sub(pattern, repl, data)
- # if this key is not in the file, append it
- if not re.match('\s*' + key + '\s*=\s*"', data):
- data += repl
+ cfg = config_file if config_file else 'config.py'
+
+ # load config file
+ with open(cfg, 'r', encoding="utf-8") as f:
+ lines = f.readlines()
+
# make sure the file ends with a carraige return
- if not re.match('\n$', data):
- data += '\n'
- with open('config.py', 'w', encoding='utf8') as f:
- f.writelines(data)
+ if len(lines) > 0:
+ if not lines[-1].endswith('\n'):
+ lines[-1] += '\n'
+
+ # regex for finding and replacing python string variable
+ # definitions/initializations
+ pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
+ repl = key + ' = "' + value + '"'
+ pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
+ repl2 = key + " = '" + value + "'"
+
+ # If we replaced this line once, we make sure won't be a
+ # second instance of this line for this key in the document.
+ didRepl = False
+ # edit config file
+ with open(cfg, 'w', encoding="utf-8") as f:
+ for line in lines:
+ if pattern.match(line) or pattern2.match(line):
+ if not didRepl:
+ line = pattern.sub(repl, line)
+ line = pattern2.sub(repl2, line)
+ f.write(line)
+ didRepl = True
+ else:
+ f.write(line)
+ if not didRepl:
+ f.write('\n')
+ f.write(repl)
+ f.write('\n')
def parse_xml(path):
def is_repo_file(filename):
'''Whether the file in a repo is a build product to be delivered to users'''
+ if isinstance(filename, str):
+ filename = filename.encode('utf-8', errors="surrogateescape")
return os.path.isfile(filename) \
- and not filename.endswith('.asc') \
- and not filename.endswith('.sig') \
+ and not filename.endswith(b'.asc') \
+ and not filename.endswith(b'.sig') \
and os.path.basename(filename) not in [
- 'index.jar',
- 'index_unsigned.jar',
- 'index.xml',
- 'index.html',
- 'index-v1.jar',
- 'index-v1.json',
- 'categories.txt',
+ b'index.jar',
+ b'index_unsigned.jar',
+ b'index.xml',
+ b'index.html',
+ b'index-v1.jar',
+ b'index-v1.json',
+ b'categories.txt',
]
~ianmdlvl / fdroidserver.git / blobdiff