From 4053f03d77e99b24bccf6f43ce63287cef410e37 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 31 May 2017 21:20:35 +0200 Subject: [PATCH] update: add --rename-apks to force APK filenames to fdroid standard uses the standard package.name_123.apk. If that exists, it appends the shasum. If that exists, then its a duplicate, so its deleted. This should help @SergeWinters with his 12,000 APKs. --- completion/bash-completion | 2 +- fdroidserver/update.py | 49 ++++++++++++++++++++++++++++++++------ tests/run-tests | 29 ++++++++++++++++++++++ tests/update.TestCase | 1 + 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index 48352447..3965791d 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -125,7 +125,7 @@ __complete_update() { opts="-c -v -q -b -i -I -e -w" lopts="--create-metadata --verbose --quiet --buildreport --interactive --icons --editor --wiki --pretty --clean --delete-unknown - --nosign --use-date-from-apk" + --nosign --rename-apks --use-date-from-apk" case "${prev}" in -e|--editor) _filedir diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b0ff8d0a..caea795b 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1089,8 +1089,14 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): """ if ' ' in apkfilename: - logging.critical("Spaces in filenames are not allowed.") - return True, None, False + if options.rename_apks: + newfilename = apkfilename.replace(' ', '_') + os.rename(os.path.join(repodir, apkfilename), + os.path.join(repodir, newfilename)) + apkfilename = newfilename + else: + logging.critical("Spaces in filenames are not allowed.") + return True, None, False apkfile = os.path.join(repodir, apkfilename) shasum = sha256sum(apkfile) @@ -1108,13 +1114,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): if not usecache: logging.debug("Processing " + apkfilename) apk = {} - apk['apkName'] = apkfilename apk['hash'] = shasum apk['hashType'] = 'sha256' - srcfilename = apkfilename[:-4] + "_src.tar.gz" - if os.path.exists(os.path.join(repodir, srcfilename)): - apk['srcname'] = srcfilename - apk['size'] = os.path.getsize(apkfile) apk['uses-permission'] = [] apk['uses-permission-sdk-23'] = [] apk['features'] = [] @@ -1147,6 +1148,35 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): logging.critical("Failed to get apk signature") return True, None, False + if options.rename_apks: + n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk' + std_short_name = os.path.join(repodir, n) + if apkfile != std_short_name: + if os.path.exists(std_short_name): + std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk') + if apkfile != std_long_name: + if os.path.exists(std_long_name): + dupdir = os.path.join('duplicates', repodir) + if not os.path.isdir(dupdir): + os.makedirs(dupdir, exist_ok=True) + dupfile = os.path.join('duplicates', std_long_name) + logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile) + os.rename(apkfile, dupfile) + return True, None, False + else: + os.rename(apkfile, std_long_name) + apkfile = std_long_name + else: + os.rename(apkfile, std_short_name) + apkfile = std_short_name + apkfilename = apkfile[len(repodir) + 1:] + + apk['apkName'] = apkfilename + srcfilename = apkfilename[:-4] + "_src.tar.gz" + if os.path.exists(os.path.join(repodir, srcfilename)): + apk['srcname'] = srcfilename + apk['size'] = os.path.getsize(apkfile) + apkzip = zipfile.ZipFile(apkfile, 'r') # if an APK has files newer than the system time, suggest updating @@ -1498,6 +1528,8 @@ def main(): help="When configured for signed indexes, create only unsigned indexes at this stage") parser.add_argument("--use-date-from-apk", action="store_true", default=False, help="Use date from apk instead of current time for newly added apks") + parser.add_argument("--rename-apks", action="store_true", default=False, + help="Rename APK files that do not match package.name_123.apk") metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W @@ -1517,6 +1549,9 @@ def main(): resize_all_icons(repodirs) sys.exit(0) + if options.rename_apks: + options.clean = True + # check that icons exist now, rather than fail at the end of `fdroid update` for k in ['repo_icon', 'archive_icon']: if k in config: diff --git a/tests/run-tests b/tests/run-tests index 84f7f766..484ed2a8 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -239,6 +239,35 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml diff $WORKSPACE/tests/repo/index.xml repo/index.xml + +#------------------------------------------------------------------------------# +echo_header 'rename apks with `fdroid update --rename-apks`' + +REPOROOT=`create_test_dir` +cd $REPOROOT +cp $WORKSPACE/tests/keystore.jks $REPOROOT/ +$fdroid init --keystore keystore.jks --repo-keyalias=sova +echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py +echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py +echo "accepted_formats = ['txt', 'yml']" >> config.py +echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py +test -d metadata || mkdir metadata +cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ +test -d repo || mkdir repo +cp $WORKSPACE/tests/urzip.apk "repo/asdfiuhk urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234 ö.apk" +$fdroid update --rename-apks +test -e repo/info.guardianproject.urzip_100.apk +cp $WORKSPACE/tests/urzip-release.apk repo/ +$fdroid update --rename-apks +test -e repo/info.guardianproject.urzip_100.apk +test -e repo/info.guardianproject.urzip_100_b4964fd.apk +cp $WORKSPACE/tests/urzip-release.apk repo/ +$fdroid update --rename-apks +test -e repo/info.guardianproject.urzip_100.apk +test -e repo/info.guardianproject.urzip_100_b4964fd.apk +test -e duplicates/repo/info.guardianproject.urzip_100_b4964fd.apk + + #------------------------------------------------------------------------------# echo_header "test metadata checks" diff --git a/tests/update.TestCase b/tests/update.TestCase index 3742f965..b5020877 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -200,6 +200,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.delete_unknown = True + fdroidserver.update.options.rename_apks = False apps = fdroidserver.metadata.read_metadata(xref=True) knownapks = fdroidserver.common.KnownApks() -- 2.30.2