from pyasn1_modules import rfc2315
from pyasn1.error import PyAsn1Error
+from distutils.util import strtobool
+
import fdroidserver.metadata
+from fdroidserver import _
+from fdroidserver.exception import FDroidException, VCSException, BuildException
from .asynchronousfilereader import AsynchronousFileReader
# A signature block file with a .DSA, .RSA, or .EC extension
CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
+APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
+STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
'r11c': None,
'r12b': "$ANDROID_NDK",
'r13b': None,
- 'r14': None,
+ 'r14b': None,
+ 'r15c': None,
},
'qt_sdk_path': None,
'build_tools': "25.0.2",
'gradle': 'gradle',
'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
+ 'allow_disabled_algorithms': False,
'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Name': 30,
- 'Summary': 80,
- 'Description': 4000,
- 'Video': 256,
- 'WhatsNew': 500,
+ 'author': 256,
+ 'name': 30,
+ 'summary': 80,
+ 'description': 4000,
+ 'video': 256,
+ 'whatsNew': 500,
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
def setup_global_opts(parser):
parser.add_argument("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
+ help=_("Spew out even more information than normal"))
parser.add_argument("-q", "--quiet", action="store_true", default=False,
- help="Restrict output to warnings and errors")
+ help=_("Restrict output to warnings and errors"))
def fill_config_defaults(thisconfig):
with io.open(config_file, "rb") as f:
code = compile(f.read(), config_file, 'exec')
exec(code, None, config)
- elif len(get_local_metadata_files()) == 0:
- logging.critical("Missing config file - is this a repo directory?")
- sys.exit(2)
+ else:
+ logging.debug("No config.py found - using defaults.")
for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
if k in config:
if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
st = os.stat(config_file)
if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
- logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
+ logging.warning("unsafe permissions on {0} (should be 0600)!".format(config_file))
fill_config_defaults(config)
def ensure_build_tools_exists(thisconfig):
if not test_sdk_exists(thisconfig):
- sys.exit(3)
+ raise FDroidException("Android SDK not found.")
build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
if not os.path.isdir(versioned_build_tools):
- logging.critical('Android Build Tools path "'
- + versioned_build_tools + '" does not exist!')
- sys.exit(3)
+ raise FDroidException(
+ 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
def get_local_metadata_files():
def read_pkg_args(args, allow_vercodes=False):
"""
- Given the arguments in the form of multiple appid:[vc] strings, this returns
- a dictionary with the set of vercodes specified for each package.
+ :param args: arguments in the form of multiple appid:[vc] strings
+ :returns: a dictionary with the set of vercodes specified for each package
"""
vercodes = {}
# Derived classes need to implement this. It's called once basic checking
# has been performend.
- def gotorevisionx(self, rev):
+ def gotorevisionx(self, rev): # pylint: disable=unused-argument
raise VCSException("This VCS type doesn't define gotorevisionx")
# Initialise and update submodules
os.path.join(app_dir, 'res'),
os.path.join(app_dir, 'src', 'main', 'res'),
]:
- for r, d, f in os.walk(res_dir):
- if os.path.basename(r) == 'values':
- xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
+ for root, dirs, files in os.walk(res_dir):
+ if os.path.basename(root) == 'values':
+ xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
name = string[len('@string/'):]
return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
-class FDroidException(Exception):
-
- def __init__(self, value, detail=None):
- self.value = value
- self.detail = detail
-
- def shortened_detail(self):
- if len(self.detail) < 16000:
- return self.detail
- return '[...]\n' + self.detail[-16000:]
-
- def get_wikitext(self):
- ret = repr(self.value) + "\n"
- if self.detail:
- ret += "=detail=\n"
- ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
- return ret
-
- def __str__(self):
- ret = self.value
- if self.detail:
- ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
- return ret
-
-
-class VCSException(FDroidException):
- pass
-
-
-class BuildException(FDroidException):
- pass
-
-
# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
-# Prepare the source code for a particular build
-# 'vcs' - the appropriate vcs object for the application
-# 'app' - the application details from the metadata
-# 'build' - the build details from the metadata
-# 'build_dir' - the path to the build directory, usually
-# 'build/app.id'
-# 'srclib_dir' - the path to the source libraries directory, usually
-# 'build/srclib'
-# 'extlib_dir' - the path to the external libraries directory, usually
-# 'build/extlib'
-# Returns the (root, srclibpaths) where:
-# 'root' is the root directory, which may be the same as 'build_dir' or may
-# be a subdirectory of it.
-# 'srclibpaths' is information on the srclibs being used
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
+ """ Prepare the source code for a particular build
+
+ :param vcs: the appropriate vcs object for the application
+ :param app: the application details from the metadata
+ :param build: the build details from the metadata
+ :param build_dir: the path to the build directory, usually 'build/app.id'
+ :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
+ :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
+
+ Returns the (root, srclibpaths) where:
+ :param root: is the root directory, which may be the same as 'build_dir' or may
+ be a subdirectory of it.
+ :param srclibpaths: is information on the srclibs being used
+ """
# Optionally, the actual app source can be in a subdirectory
if build.subdir:
class KnownApks:
+ """permanent store of existing APKs with the date they were added
+
+ This is currently the only way to permanently store the "updated"
+ date of APKs.
+ """
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
- def recordapk(self, apk, app, default_date=None):
+ def recordapk(self, apkName, app, default_date=None):
'''
Record an apk (if it's new, otherwise does nothing)
Returns the date it was added as a datetime instance
'''
- if apk not in self.apks:
+ if apkName not in self.apks:
if default_date is None:
default_date = datetime.utcnow()
- self.apks[apk] = (app, default_date)
+ self.apks[apkName] = (app, default_date)
self.changed = True
- _, added = self.apks[apk]
+ _, added = self.apks[apkName]
return added
# Look up information - given the 'apkname', returns (app id, date added/None).
return os.path.splitext(filename)[1].lower()[1:]
-def isApkAndDebuggable(apkfile, config):
+def get_apk_debuggable_aapt(apkfile):
+ p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
+ output=False)
+ if p.returncode != 0:
+ raise FDroidException("Failed to get apk manifest information")
+ for line in p.output.splitlines():
+ if 'android:debuggable' in line and not line.endswith('0x0'):
+ return True
+ return False
+
+
+def get_apk_debuggable_androguard(apkfile):
+ try:
+ from androguard.core.bytecodes.apk import APK
+ except ImportError:
+ raise FDroidException("androguard library is not installed and aapt not present")
+
+ apkobject = APK(apkfile)
+ if apkobject.is_valid_APK():
+ debuggable = apkobject.get_element("application", "debuggable")
+ if debuggable is not None:
+ return bool(strtobool(debuggable))
+ return False
+
+
+def isApkAndDebuggable(apkfile):
"""Returns True if the given file is an APK and is debuggable
:param apkfile: full path to the apk to check"""
if get_file_extension(apkfile) != 'apk':
return False
- p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
- output=False)
- if p.returncode != 0:
- logging.critical("Failed to get apk manifest information")
- sys.exit(1)
+ if SdkToolsPopen(['aapt', 'version'], output=False):
+ return get_apk_debuggable_aapt(apkfile)
+ else:
+ return get_apk_debuggable_androguard(apkfile)
+
+
+def get_apk_id_aapt(apkfile):
+ """Extrat identification information from APK using aapt.
+
+ :param apkfile: path to an APK file.
+ :returns: triplet (appid, version code, version name)
+ """
+ r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
+ p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
for line in p.output.splitlines():
- if 'android:debuggable' in line and not line.endswith('0x0'):
- return True
- return False
+ m = r.match(line)
+ if m:
+ return m.group('appid'), m.group('vercode'), m.group('vername')
+ raise FDroidException("reading identification failed, APK invalid: '{}'".format(apkfile))
class PopenResult:
config[cmd] = find_sdk_tools_cmd(commands[0])
abscmd = config[cmd]
if abscmd is None:
- logging.critical("Could not find '%s' on your system" % cmd)
- sys.exit(1)
+ raise FDroidException("Could not find '%s' on your system" % cmd)
if cmd == 'aapt':
test_aapt_version(config['aapt'])
return FDroidPopen([abscmd] + commands[1:],
result.returncode = p.wait()
result.output = buf.getvalue()
buf.close()
+ # make sure all filestreams of the subprocess are closed
+ for streamvar in ['stdin', 'stdout', 'stderr']:
+ if hasattr(p, streamvar):
+ stream = getattr(p, streamvar)
+ if stream:
+ stream.close()
return result
apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
+def metadata_get_sigdir(appid, vercode=None):
+ """Get signature directory for app"""
+ if vercode:
+ return os.path.join('metadata', appid, 'signatures', vercode)
+ else:
+ return os.path.join('metadata', appid, 'signatures')
+
+
+def apk_extract_signatures(apkpath, outdir, manifest=True):
+ """Extracts a signature files from APK and puts them into target directory.
+
+ :param apkpath: location of the apk
+ :param outdir: folder where the extracted signature files will be stored
+ :param manifest: (optionally) disable extracting manifest file
+ """
+ with ZipFile(apkpath, 'r') as in_apk:
+ for f in in_apk.infolist():
+ if apk_sigfile.match(f.filename) or \
+ (manifest and f.filename == 'META-INF/MANIFEST.MF'):
+ newpath = os.path.join(outdir, os.path.basename(f.filename))
+ with open(newpath, 'wb') as out_file:
+ out_file.write(in_apk.read(f.filename))
+
+
def verify_apks(signed_apk, unsigned_apk, tmp_dir):
"""Verify that two apks are the same
if not verified:
logging.info("...NOT verified - {0}".format(tmp_apk))
- return compare_apks(signed_apk, tmp_apk, tmp_dir, os.path.dirname(unsigned_apk))
+ return compare_apks(signed_apk, tmp_apk, tmp_dir,
+ os.path.dirname(unsigned_apk))
logging.info("...successfully verified")
return None
return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
+def verify_old_apk_signature(apk):
+ """verify the signature on an archived APK, supporting deprecated algorithms
+
+ F-Droid aims to keep every single binary that it ever published. Therefore,
+ it needs to be able to verify APK signatures that include deprecated/removed
+ algorithms. For example, jarsigner treats an MD5 signature as unsigned.
+
+ jarsigner passes unsigned APKs as "verified"! So this has to turn
+ on -strict then check for result 4.
+
+ """
+
+ _java_security = os.path.join(os.getcwd(), '.java.security')
+ with open(_java_security, 'w') as fp:
+ fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
+
+ return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
+ '-strict', '-verify', apk]) == 4
+
+
apk_badchars = re.compile('''[/ :;'"]''')
p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
lines = p.output.splitlines()
if len(lines) != 1 or 'META-INF' not in lines[0]:
- meld = find_command('meld')
- if meld is not None:
- p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
+ if set_command_in_config('meld'):
+ p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
return("Unexpected diff output - " + p.output)
# since everything verifies, delete the comparison to keep cruft down
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
cfg = config_file if config_file else 'config.py'
- # load config file
+ # load config file, create one if it doesn't exist
+ if not os.path.exists(cfg):
+ os.mknod(cfg)
+ logging.info("Creating empty " + cfg)
with open(cfg, 'r', encoding="utf-8") as f:
lines = f.readlines()
~ianmdlvl / fdroidserver.git / blobdiff