3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
76 all_screen_densities = ['0'] + screen_densities
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 def dpi_to_px(density):
88 return (int(density) * 48) / 160
92 return (int(px) * 160) / 48
95 def get_icon_dir(repodir, density):
97 return os.path.join(repodir, "icons")
98 return os.path.join(repodir, "icons-%s" % density)
101 def get_icon_dirs(repodir):
102 for density in screen_densities:
103 yield get_icon_dir(repodir, density)
106 def get_all_icon_dirs(repodir):
107 for density in all_screen_densities:
108 yield get_icon_dir(repodir, density)
111 def update_wiki(apps, sortedids, apks):
114 :param apps: fully populated list of all applications
115 :param apks: all apks, except...
117 logging.info("Updating wiki")
119 wikiredircat = 'App Redirects'
121 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122 path=config['wiki_path'])
123 site.login(config['wiki_user'], config['wiki_password'])
125 generated_redirects = {}
127 for appid in sortedids:
128 app = metadata.App(apps[appid])
132 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
134 for af in app.AntiFeatures:
135 wikidata += '{{AntiFeature|' + af + '}}\n'
140 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
143 app.added.strftime('%Y-%m-%d') if app.added else '',
144 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
159 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
161 wikidata += app.Summary
162 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
164 wikidata += "=Description=\n"
165 wikidata += metadata.description_wiki(app.Description) + "\n"
167 wikidata += "=Maintainer Notes=\n"
168 if app.MaintainerNotes:
169 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
172 # Get a list of all packages for this application...
174 gotcurrentver = False
178 if apk['packageName'] == appid:
179 if str(apk['versionCode']) == app.CurrentVersionCode:
182 # Include ones we can't build, as a special case...
183 for build in app.builds:
185 if build.versionCode == app.CurrentVersionCode:
187 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188 apklist.append({'versionCode': int(build.versionCode),
189 'versionName': build.versionName,
190 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
195 if apk['versionCode'] == int(build.versionCode):
200 apklist.append({'versionCode': int(build.versionCode),
201 'versionName': build.versionName,
202 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
204 if app.CurrentVersionCode == '0':
206 # Sort with most recent first...
207 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
209 wikidata += "=Versions=\n"
210 if len(apklist) == 0:
211 wikidata += "We currently have no versions of this app available."
212 elif not gotcurrentver:
213 wikidata += "We don't have the current version of this app."
215 wikidata += "We have the current version of this app."
216 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218 if len(app.NoSourceSince) > 0:
219 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220 if len(app.CurrentVersion) > 0:
221 wikidata += "The current (recommended) version is " + app.CurrentVersion
222 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
225 wikidata += "==" + apk['versionName'] + "==\n"
227 if 'buildproblem' in apk:
228 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
231 wikidata += "This version is built and signed by "
233 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
235 wikidata += "the original developer.\n\n"
236 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
238 wikidata += '\n[[Category:' + wikicat + ']]\n'
239 if len(app.NoSourceSince) > 0:
240 wikidata += '\n[[Category:Apps missing source code]]\n'
241 if validapks == 0 and not app.Disabled:
242 wikidata += '\n[[Category:Apps with no packages]]\n'
243 if cantupdate and not app.Disabled:
244 wikidata += "\n[[Category:Apps we cannot update]]\n"
245 if buildfails and not app.Disabled:
246 wikidata += "\n[[Category:Apps with failing builds]]\n"
247 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248 wikidata += '\n[[Category:Apps to Update]]\n'
250 wikidata += '\n[[Category:Apps that are disabled]]\n'
251 if app.UpdateCheckMode == 'None' and not app.Disabled:
252 wikidata += '\n[[Category:Apps with no update check]]\n'
253 for appcat in app.Categories:
254 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
256 # We can't have underscores in the page name, even if they're in
257 # the package ID, because MediaWiki messes with them...
258 pagename = appid.replace('_', ' ')
260 # Drop a trailing newline, because mediawiki is going to drop it anyway
261 # and it we don't we'll think the page has changed when it hasn't...
262 if wikidata.endswith('\n'):
263 wikidata = wikidata[:-1]
265 generated_pages[pagename] = wikidata
267 # Make a redirect from the name to the ID too, unless there's
268 # already an existing page with the name and it isn't a redirect.
270 apppagename = app.Name.replace('_', ' ')
271 apppagename = apppagename.replace('{', '')
272 apppagename = apppagename.replace('}', ' ')
273 apppagename = apppagename.replace(':', ' ')
274 apppagename = apppagename.replace('[', ' ')
275 apppagename = apppagename.replace(']', ' ')
276 # Drop double spaces caused mostly by replacing ':' above
277 apppagename = apppagename.replace(' ', ' ')
278 for expagename in site.allpages(prefix=apppagename,
279 filterredir='nonredirects',
281 if expagename == apppagename:
283 # Another reason not to make the redirect page is if the app name
284 # is the same as it's ID, because that will overwrite the real page
285 # with an redirect to itself! (Although it seems like an odd
286 # scenario this happens a lot, e.g. where there is metadata but no
287 # builds or binaries to extract a name from.
288 if apppagename == pagename:
291 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
293 for tcat, genp in [(wikicat, generated_pages),
294 (wikiredircat, generated_redirects)]:
295 catpages = site.Pages['Category:' + tcat]
297 for page in catpages:
298 existingpages.append(page.name)
299 if page.name in genp:
300 pagetxt = page.edit()
301 if pagetxt != genp[page.name]:
302 logging.debug("Updating modified page " + page.name)
303 page.save(genp[page.name], summary='Auto-updated')
305 logging.debug("Page " + page.name + " is unchanged")
307 logging.warn("Deleting page " + page.name)
308 page.delete('No longer published')
309 for pagename, text in genp.items():
310 logging.debug("Checking " + pagename)
311 if pagename not in existingpages:
312 logging.debug("Creating page " + pagename)
314 newpage = site.Pages[pagename]
315 newpage.save(text, summary='Auto-created')
316 except Exception as e:
317 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
319 # Purge server cache to ensure counts are up to date
320 site.pages['Repository Maintenance'].purge()
323 def delete_disabled_builds(apps, apkcache, repodirs):
324 """Delete disabled build outputs.
326 :param apps: list of all applications, as per metadata.read_metadata
327 :param apkcache: current apk cache information
328 :param repodirs: the repo directories to process
330 for appid, app in apps.items():
331 for build in app['builds']:
332 if not build.disable:
334 apkfilename = common.get_release_filename(app, build)
335 iconfilename = "%s.%s.png" % (
338 for repodir in repodirs:
340 os.path.join(repodir, apkfilename),
341 os.path.join(repodir, apkfilename + '.asc'),
342 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
344 for density in all_screen_densities:
345 repo_dir = get_icon_dir(repodir, density)
346 files.append(os.path.join(repo_dir, iconfilename))
349 if os.path.exists(f):
350 logging.info("Deleting disabled build output " + f)
352 if apkfilename in apkcache:
353 del apkcache[apkfilename]
356 def resize_icon(iconpath, density):
358 if not os.path.isfile(iconpath):
363 fp = open(iconpath, 'rb')
365 size = dpi_to_px(density)
367 if any(length > size for length in im.size):
369 im.thumbnail((size, size), Image.ANTIALIAS)
370 logging.debug("%s was too large at %s - new size is %s" % (
371 iconpath, oldsize, im.size))
372 im.save(iconpath, "PNG")
374 except Exception as e:
375 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
382 def resize_all_icons(repodirs):
383 """Resize all icons that exceed the max size
385 :param repodirs: the repo directories to process
387 for repodir in repodirs:
388 for density in screen_densities:
389 icon_dir = get_icon_dir(repodir, density)
390 icon_glob = os.path.join(icon_dir, '*.png')
391 for iconpath in glob.glob(icon_glob):
392 resize_icon(iconpath, density)
396 """ Get the signing certificate of an apk. To get the same md5 has that
397 Android gets, we encode the .RSA certificate in a specific format and pass
398 it hex-encoded to the md5 digest algorithm.
400 :param apkpath: path to the apk
401 :returns: A string containing the md5 of the signature of the apk or None
402 if an error occurred.
405 with zipfile.ZipFile(apkpath, 'r') as apk:
406 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
409 logging.error("Found no signing certificates on %s" % apkpath)
412 logging.error("Found multiple signing certificates on %s" % apkpath)
415 cert = apk.read(certs[0])
417 cert_encoded = common.get_certificate(cert)
419 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
422 def get_cache_file():
423 return os.path.join('tmp', 'apkcache')
427 """Get the cached dict of the APK index
429 Gather information about all the apk files in the repo directory,
430 using cached data if possible. Some of the index operations take a
431 long time, like calculating the SHA-256 and verifying the APK
434 The cache is invalidated if the metadata version is different, or
435 the 'allow_disabled_algorithms' config/option is different. In
436 those cases, there is no easy way to know what has changed from
437 the cache, so just rerun the whole thing.
442 apkcachefile = get_cache_file()
443 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
444 if not options.clean and os.path.exists(apkcachefile):
445 with open(apkcachefile, 'rb') as cf:
446 apkcache = pickle.load(cf, encoding='utf-8')
447 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
448 or apkcache.get('allow_disabled_algorithms') != ada:
453 apkcache["METADATA_VERSION"] = METADATA_VERSION
454 apkcache['allow_disabled_algorithms'] = ada
459 def write_cache(apkcache):
460 apkcachefile = get_cache_file()
461 cache_path = os.path.dirname(apkcachefile)
462 if not os.path.exists(cache_path):
463 os.makedirs(cache_path)
464 with open(apkcachefile, 'wb') as cf:
465 pickle.dump(apkcache, cf)
468 def get_icon_bytes(apkzip, iconsrc):
469 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
471 return apkzip.read(iconsrc)
473 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
476 def sha256sum(filename):
477 '''Calculate the sha256 of the given file'''
478 sha = hashlib.sha256()
479 with open(filename, 'rb') as f:
485 return sha.hexdigest()
488 def has_known_vulnerability(filename):
489 """checks for known vulnerabilities in the APK
491 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
492 version. Google also enforces this:
493 https://support.google.com/faqs/answer/6376725?hl=en
495 Checks whether there are more than one classes.dex or AndroidManifest.xml
496 files, which is invalid and an essential part of the "Master Key" attack.
498 http://www.saurik.com/id/17
501 # statically load this pattern
502 if not hasattr(has_known_vulnerability, "pattern"):
503 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
506 with zipfile.ZipFile(filename) as zf:
507 for name in zf.namelist():
508 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
511 chunk = lib.read(4096)
514 m = has_known_vulnerability.pattern.search(chunk)
516 version = m.group(1).decode('ascii')
517 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
518 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
519 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
520 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
522 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
525 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
526 if name in files_in_apk:
528 files_in_apk.add(name)
533 def insert_obbs(repodir, apps, apks):
534 """Scans the .obb files in a given repo directory and adds them to the
535 relevant APK instances. OBB files have versionCodes like APK
536 files, and they are loosely associated. If there is an OBB file
537 present, then any APK with the same or higher versionCode will use
538 that OBB file. There are two OBB types: main and patch, each APK
539 can only have only have one of each.
541 https://developer.android.com/google/play/expansion-files.html
543 :param repodir: repo directory to scan
544 :param apps: list of current, valid apps
545 :param apks: current information on all APKs
549 def obbWarnDelete(f, msg):
550 logging.warning(msg + f)
551 if options.delete_unknown:
552 logging.error("Deleting unknown file: " + f)
556 java_Integer_MIN_VALUE = -pow(2, 31)
557 currentPackageNames = apps.keys()
558 for f in glob.glob(os.path.join(repodir, '*.obb')):
559 obbfile = os.path.basename(f)
560 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
561 chunks = obbfile.split('.')
562 if chunks[0] != 'main' and chunks[0] != 'patch':
563 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
565 if not re.match(r'^-?[0-9]+$', chunks[1]):
566 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
568 versionCode = int(chunks[1])
569 packagename = ".".join(chunks[2:-1])
571 highestVersionCode = java_Integer_MIN_VALUE
572 if packagename not in currentPackageNames:
573 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
576 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
577 highestVersionCode = apk['versionCode']
578 if versionCode > highestVersionCode:
579 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
580 + ') than any APK: ')
582 obbsha256 = sha256sum(f)
583 obbs.append((packagename, versionCode, obbfile, obbsha256))
586 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
587 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
588 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
589 apk['obbMainFile'] = obbfile
590 apk['obbMainFileSha256'] = obbsha256
591 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
592 apk['obbPatchFile'] = obbfile
593 apk['obbPatchFileSha256'] = obbsha256
594 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
598 def translate_per_build_anti_features(apps, apks):
599 """Grab the anti-features list from the build metadata
601 For most Anti-Features, they are really most applicable per-APK,
602 not for an app. An app can fix a vulnerability, add/remove
603 tracking, etc. This reads the 'antifeatures' list from the Build
604 entries in the fdroiddata metadata file, then transforms it into
605 the 'antiFeatures' list of unique items for the index.
607 The field key is all lower case in the metadata file to match the
608 rest of the Build fields. It is 'antiFeatures' camel case in the
609 implementation, index, and fdroidclient since it is translated
610 from the build 'antifeatures' field, not directly included.
614 antiFeatures = dict()
615 for packageName, app in apps.items():
617 for build in app['builds']:
618 afl = build.get('antifeatures')
620 d[int(build.versionCode)] = afl
622 antiFeatures[packageName] = d
625 d = antiFeatures.get(apk['packageName'])
627 afl = d.get(apk['versionCode'])
629 apk['antiFeatures'].update(afl)
632 def _get_localized_dict(app, locale):
633 '''get the dict to add localized store metadata to'''
634 if 'localized' not in app:
635 app['localized'] = collections.OrderedDict()
636 if locale not in app['localized']:
637 app['localized'][locale] = collections.OrderedDict()
638 return app['localized'][locale]
641 def _set_localized_text_entry(app, locale, key, f):
642 limit = config['char_limits'][key]
643 localized = _get_localized_dict(app, locale)
645 text = fp.read()[:limit]
647 localized[key] = text
650 def _set_author_entry(app, key, f):
651 limit = config['char_limits']['author']
653 text = fp.read()[:limit]
658 def copy_triple_t_store_metadata(apps):
659 """Include store metadata from the app's source repo
661 The Triple-T Gradle Play Publisher is a plugin that has a standard
662 file layout for all of the metadata and graphics that the Google
663 Play Store accepts. Since F-Droid has the git repo, it can just
664 pluck those files directly. This method reads any text files into
665 the app dict, then copies any graphics into the fdroid repo
668 This needs to be run before insert_localized_app_metadata() so that
669 the graphics files that are copied into the fdroid repo get
672 https://github.com/Triple-T/gradle-play-publisher#upload-images
673 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
677 if not os.path.isdir('build'):
678 return # nothing to do
680 for packageName, app in apps.items():
681 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
682 logging.debug('Triple-T Gradle Play Publisher: ' + d)
683 for root, dirs, files in os.walk(d):
684 segments = root.split('/')
685 locale = segments[-2]
687 if f == 'fulldescription':
688 _set_localized_text_entry(app, locale, 'description',
689 os.path.join(root, f))
691 elif f == 'shortdescription':
692 _set_localized_text_entry(app, locale, 'summary',
693 os.path.join(root, f))
696 _set_localized_text_entry(app, locale, 'name',
697 os.path.join(root, f))
700 _set_localized_text_entry(app, locale, 'video',
701 os.path.join(root, f))
703 elif f == 'whatsnew':
704 _set_localized_text_entry(app, segments[-1], 'whatsNew',
705 os.path.join(root, f))
707 elif f == 'contactEmail':
708 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
710 elif f == 'contactPhone':
711 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
713 elif f == 'contactWebsite':
714 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
717 base, extension = common.get_extension(f)
718 dirname = os.path.basename(root)
719 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
720 if segments[-2] == 'listing':
721 locale = segments[-3]
723 locale = segments[-2]
724 destdir = os.path.join('repo', packageName, locale)
725 os.makedirs(destdir, mode=0o755, exist_ok=True)
726 sourcefile = os.path.join(root, f)
727 destfile = os.path.join(destdir, dirname + '.' + extension)
728 logging.debug('copying ' + sourcefile + ' ' + destfile)
729 shutil.copy(sourcefile, destfile)
732 def insert_localized_app_metadata(apps):
733 """scans standard locations for graphics and localized text
735 Scans for localized description files, store graphics, and
736 screenshot PNG files in statically defined screenshots directory
737 and adds them to the app metadata. The screenshots and graphic
738 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
739 and must be in the following layout:
740 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
742 repo/packageName/locale/featureGraphic.png
743 repo/packageName/locale/phoneScreenshots/1.png
744 repo/packageName/locale/phoneScreenshots/2.png
746 The changelog files must be text files named with the versionCode
747 ending with ".txt" and must be in the following layout:
748 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
750 repo/packageName/locale/changelogs/12345.txt
752 This will scan the each app's source repo then the metadata/ dir
753 for these standard locations of changelog files. If it finds
754 them, they will be added to the dict of all packages, with the
755 versions in the metadata/ folder taking precendence over the what
756 is in the app's source repo.
758 Where "packageName" is the app's packageName and "locale" is the locale
759 of the graphics, e.g. what language they are in, using the IETF RFC5646
760 format (en-US, fr-CA, es-MX, etc).
762 This will also scan the app's git for a fastlane folder, and the
763 metadata/ folder and the apps' source repos for standard locations
764 of graphic and screenshot files. If it finds them, it will copy
765 them into the repo. The fastlane files follow this pattern:
766 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
770 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
771 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
772 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
774 for srcd in sorted(sourcedirs):
775 if not os.path.isdir(srcd):
777 for root, dirs, files in os.walk(srcd):
778 segments = root.split('/')
779 packageName = segments[1]
780 if packageName not in apps:
781 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
783 locale = segments[-1]
784 destdir = os.path.join('repo', packageName, locale)
786 if f in ('description.txt', 'full_description.txt'):
787 _set_localized_text_entry(apps[packageName], locale, 'description',
788 os.path.join(root, f))
790 elif f in ('summary.txt', 'short_description.txt'):
791 _set_localized_text_entry(apps[packageName], locale, 'summary',
792 os.path.join(root, f))
794 elif f in ('name.txt', 'title.txt'):
795 _set_localized_text_entry(apps[packageName], locale, 'name',
796 os.path.join(root, f))
798 elif f == 'video.txt':
799 _set_localized_text_entry(apps[packageName], locale, 'video',
800 os.path.join(root, f))
802 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
803 locale = segments[-2]
804 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
805 os.path.join(root, f))
808 base, extension = common.get_extension(f)
809 if locale == 'images':
810 locale = segments[-2]
811 destdir = os.path.join('repo', packageName, locale)
812 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
813 os.makedirs(destdir, mode=0o755, exist_ok=True)
814 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
815 shutil.copy(os.path.join(root, f), destdir)
817 if d in SCREENSHOT_DIRS:
818 for f in glob.glob(os.path.join(root, d, '*.*')):
819 _, extension = common.get_extension(f)
820 if extension in ALLOWED_EXTENSIONS:
821 screenshotdestdir = os.path.join(destdir, d)
822 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
823 logging.debug('copying ' + f + ' ' + screenshotdestdir)
824 shutil.copy(f, screenshotdestdir)
826 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
828 if not os.path.isdir(d):
830 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
831 if not os.path.isfile(f):
833 segments = f.split('/')
834 packageName = segments[1]
836 screenshotdir = segments[3]
837 filename = os.path.basename(f)
838 base, extension = common.get_extension(filename)
840 if packageName not in apps:
841 logging.warning('Found "%s" graphic without metadata for app "%s"!'
842 % (filename, packageName))
844 graphics = _get_localized_dict(apps[packageName], locale)
846 if extension not in ALLOWED_EXTENSIONS:
847 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
848 elif base in GRAPHIC_NAMES:
849 # there can only be zero or one of these per locale
850 graphics[base] = filename
851 elif screenshotdir in SCREENSHOT_DIRS:
852 # there can any number of these per locale
853 logging.debug('adding to ' + screenshotdir + ': ' + f)
854 if screenshotdir not in graphics:
855 graphics[screenshotdir] = []
856 graphics[screenshotdir].append(filename)
858 logging.warning('Unsupported graphics file found: ' + f)
861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
862 """Scan a repo for all files with an extension except APK/OBB
864 :param apkcache: current cached info about all repo files
865 :param repodir: repo directory to scan
866 :param knownapks: list of all known files, as per metadata.read_metadata
867 :param use_date_from_file: use date from file (instead of current date)
868 for newly added files
873 repodir = repodir.encode('utf-8')
874 for name in os.listdir(repodir):
875 file_extension = common.get_file_extension(name)
876 if file_extension == 'apk' or file_extension == 'obb':
878 filename = os.path.join(repodir, name)
879 name_utf8 = name.decode('utf-8')
880 if filename.endswith(b'_src.tar.gz'):
881 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
883 if not common.is_repo_file(filename):
885 stat = os.stat(filename)
886 if stat.st_size == 0:
887 raise FDroidException(filename + ' is zero size!')
889 shasum = sha256sum(filename)
892 repo_file = apkcache[name]
893 # added time is cached as tuple but used here as datetime instance
894 if 'added' in repo_file:
895 a = repo_file['added']
896 if isinstance(a, datetime):
897 repo_file['added'] = a
899 repo_file['added'] = datetime(*a[:6])
900 if repo_file.get('hash') == shasum:
901 logging.debug("Reading " + name_utf8 + " from cache")
904 logging.debug("Ignoring stale cache data for " + name)
907 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
908 repo_file = collections.OrderedDict()
909 repo_file['name'] = os.path.splitext(name_utf8)[0]
910 # TODO rename apkname globally to something more generic
911 repo_file['apkName'] = name_utf8
912 repo_file['hash'] = shasum
913 repo_file['hashType'] = 'sha256'
914 repo_file['versionCode'] = 0
915 repo_file['versionName'] = shasum
916 # the static ID is the SHA256 unless it is set in the metadata
917 repo_file['packageName'] = shasum
919 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
921 repo_file['packageName'] = m.group(1)
922 repo_file['versionCode'] = int(m.group(2))
923 srcfilename = name + b'_src.tar.gz'
924 if os.path.exists(os.path.join(repodir, srcfilename)):
925 repo_file['srcname'] = srcfilename.decode('utf-8')
926 repo_file['size'] = stat.st_size
928 apkcache[name] = repo_file
931 if use_date_from_file:
932 timestamp = stat.st_ctime
933 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
935 default_date_param = None
937 # Record in knownapks, getting the added date at the same time..
938 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
939 default_date=default_date_param)
941 repo_file['added'] = added
943 repo_files.append(repo_file)
945 return repo_files, cachechanged
948 def scan_apk(apk_file):
950 Scans an APK file and returns dictionary with metadata of the APK.
952 Attention: This does *not* verify that the APK signature is correct.
954 :param apk_file: The (ideally absolute) path to the APK file
955 :raises BuildException
956 :return A dict containing APK metadata
959 'hash': sha256sum(apk_file),
960 'hashType': 'sha256',
961 'uses-permission': [],
962 'uses-permission-sdk-23': [],
966 'antiFeatures': set(),
969 if SdkToolsPopen(['aapt', 'version'], output=False):
970 scan_apk_aapt(apk, apk_file)
972 scan_apk_androguard(apk, apk_file)
974 # Get the signature, or rather the signing key fingerprints
975 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
976 apk['sig'] = getsig(apk_file)
978 raise BuildException("Failed to get apk signature")
979 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
981 if not apk.get('signer'):
982 raise BuildException("Failed to get apk signing key fingerprint")
984 # Get size of the APK
985 apk['size'] = os.path.getsize(apk_file)
987 if 'minSdkVersion' not in apk:
988 logging.warning("No SDK version information found in {0}".format(apk_file))
989 apk['minSdkVersion'] = 1
991 # Check for known vulnerabilities
992 if has_known_vulnerability(apk_file):
993 apk['antiFeatures'].add('KnownVuln')
998 def scan_apk_aapt(apk, apkfile):
999 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1000 if p.returncode != 0:
1001 if options.delete_unknown:
1002 if os.path.exists(apkfile):
1003 logging.error("Failed to get apk information, deleting " + apkfile)
1006 logging.error("Could not find {0} to remove it".format(apkfile))
1008 logging.error("Failed to get apk information, skipping " + apkfile)
1009 raise BuildException("Invalid APK")
1010 for line in p.output.splitlines():
1011 if line.startswith("package:"):
1013 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1014 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1015 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1016 except Exception as e:
1017 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1018 elif line.startswith("application:"):
1019 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1020 # Keep path to non-dpi icon in case we need it
1021 match = re.match(APK_ICON_PAT_NODPI, line)
1023 apk['icons_src']['-1'] = match.group(1)
1024 elif line.startswith("launchable-activity:"):
1025 # Only use launchable-activity as fallback to application
1027 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1028 if '-1' not in apk['icons_src']:
1029 match = re.match(APK_ICON_PAT_NODPI, line)
1031 apk['icons_src']['-1'] = match.group(1)
1032 elif line.startswith("application-icon-"):
1033 match = re.match(APK_ICON_PAT, line)
1035 density = match.group(1)
1036 path = match.group(2)
1037 apk['icons_src'][density] = path
1038 elif line.startswith("sdkVersion:"):
1039 m = re.match(APK_SDK_VERSION_PAT, line)
1041 logging.error(line.replace('sdkVersion:', '')
1042 + ' is not a valid minSdkVersion!')
1044 apk['minSdkVersion'] = m.group(1)
1045 # if target not set, default to min
1046 if 'targetSdkVersion' not in apk:
1047 apk['targetSdkVersion'] = m.group(1)
1048 elif line.startswith("targetSdkVersion:"):
1049 m = re.match(APK_SDK_VERSION_PAT, line)
1051 logging.error(line.replace('targetSdkVersion:', '')
1052 + ' is not a valid targetSdkVersion!')
1054 apk['targetSdkVersion'] = m.group(1)
1055 elif line.startswith("maxSdkVersion:"):
1056 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1057 elif line.startswith("native-code:"):
1058 apk['nativecode'] = []
1059 for arch in line[13:].split(' '):
1060 apk['nativecode'].append(arch[1:-1])
1061 elif line.startswith('uses-permission:'):
1062 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1063 if perm_match['maxSdkVersion']:
1064 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1065 permission = UsesPermission(
1067 perm_match['maxSdkVersion']
1070 apk['uses-permission'].append(permission)
1071 elif line.startswith('uses-permission-sdk-23:'):
1072 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1073 if perm_match['maxSdkVersion']:
1074 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1075 permission_sdk_23 = UsesPermissionSdk23(
1077 perm_match['maxSdkVersion']
1080 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1082 elif line.startswith('uses-feature:'):
1083 feature = re.match(APK_FEATURE_PAT, line).group(1)
1084 # Filter out this, it's only added with the latest SDK tools and
1085 # causes problems for lots of apps.
1086 if feature != "android.hardware.screen.portrait" \
1087 and feature != "android.hardware.screen.landscape":
1088 if feature.startswith("android.feature."):
1089 feature = feature[16:]
1090 apk['features'].add(feature)
1093 def scan_apk_androguard(apk, apkfile):
1095 from androguard.core.bytecodes.apk import APK
1096 apkobject = APK(apkfile)
1097 if apkobject.is_valid_APK():
1098 arsc = apkobject.get_android_resources()
1100 if options.delete_unknown:
1101 if os.path.exists(apkfile):
1102 logging.error("Failed to get apk information, deleting " + apkfile)
1105 logging.error("Could not find {0} to remove it".format(apkfile))
1107 logging.error("Failed to get apk information, skipping " + apkfile)
1108 raise BuildException("Invaild APK")
1110 raise FDroidException("androguard library is not installed and aapt not present")
1111 except FileNotFoundError:
1112 logging.error("Could not open apk file for analysis")
1113 raise BuildException("Invalid APK")
1115 apk['packageName'] = apkobject.get_package()
1116 apk['versionCode'] = int(apkobject.get_androidversion_code())
1117 apk['versionName'] = apkobject.get_androidversion_name()
1118 if apk['versionName'][0] == "@":
1119 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1120 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1121 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1122 apk['name'] = apkobject.get_app_name()
1124 if apkobject.get_max_sdk_version() is not None:
1125 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1126 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1127 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1129 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1130 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1132 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1134 for file in apkobject.get_files():
1135 d_re = density_re.match(file)
1137 folder = d_re.group(1).split('-')
1139 resolution = folder[1]
1142 density = screen_resolutions[resolution]
1143 apk['icons_src'][density] = d_re.group(0)
1145 if apk['icons_src'].get('-1') is None:
1146 apk['icons_src']['-1'] = apk['icons_src']['160']
1148 arch_re = re.compile("^lib/(.*)/.*$")
1149 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1151 apk['nativecode'] = []
1152 apk['nativecode'].extend(sorted(list(arch)))
1154 xml = apkobject.get_android_manifest_xml()
1156 for item in xml.getElementsByTagName('uses-permission'):
1157 name = str(item.getAttribute("android:name"))
1158 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1159 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1160 permission = UsesPermission(
1164 apk['uses-permission'].append(permission)
1166 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1167 name = str(item.getAttribute("android:name"))
1168 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1169 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1170 permission_sdk_23 = UsesPermissionSdk23(
1174 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1176 for item in xml.getElementsByTagName('uses-feature'):
1177 feature = str(item.getAttribute("android:name"))
1178 if feature != "android.hardware.screen.portrait" \
1179 and feature != "android.hardware.screen.landscape":
1180 if feature.startswith("android.feature."):
1181 feature = feature[16:]
1182 apk['features'].append(feature)
1185 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1186 allow_disabled_algorithms=False, archive_bad_sig=False):
1187 """Processes the apk with the given filename in the given repo directory.
1189 This also extracts the icons.
1191 :param apkcache: current apk cache information
1192 :param apkfilename: the filename of the apk to scan
1193 :param repodir: repo directory to scan
1194 :param knownapks: known apks info
1195 :param use_date_from_apk: use date from APK (instead of current date)
1196 for newly added APKs
1197 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1198 disabled algorithms in the signature (e.g. MD5)
1199 :param archive_bad_sig: move APKs with a bad signature to the archive
1200 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1201 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1205 apkfile = os.path.join(repodir, apkfilename)
1207 cachechanged = False
1209 if apkfilename in apkcache:
1210 apk = apkcache[apkfilename]
1211 if apk.get('hash') == sha256sum(apkfile):
1212 logging.debug("Reading " + apkfilename + " from cache")
1215 logging.debug("Ignoring stale cache data for " + apkfilename)
1218 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1221 apk = scan_apk(apkfile)
1222 except BuildException:
1223 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1224 .format(apkfilename=apkfilename))
1225 return True, None, False
1227 # Check for debuggable apks...
1228 if common.isApkAndDebuggable(apkfile):
1229 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1231 if options.rename_apks:
1232 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1233 std_short_name = os.path.join(repodir, n)
1234 if apkfile != std_short_name:
1235 if os.path.exists(std_short_name):
1236 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1237 if apkfile != std_long_name:
1238 if os.path.exists(std_long_name):
1239 dupdir = os.path.join('duplicates', repodir)
1240 if not os.path.isdir(dupdir):
1241 os.makedirs(dupdir, exist_ok=True)
1242 dupfile = os.path.join('duplicates', std_long_name)
1243 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1244 os.rename(apkfile, dupfile)
1245 return True, None, False
1247 os.rename(apkfile, std_long_name)
1248 apkfile = std_long_name
1250 os.rename(apkfile, std_short_name)
1251 apkfile = std_short_name
1252 apkfilename = apkfile[len(repodir) + 1:]
1254 apk['apkName'] = apkfilename
1255 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1256 if os.path.exists(os.path.join(repodir, srcfilename)):
1257 apk['srcname'] = srcfilename
1259 # verify the jar signature is correct, allow deprecated
1260 # algorithms only if the APK is in the archive.
1262 if not common.verify_apk_signature(apkfile):
1263 if repodir == 'archive' or allow_disabled_algorithms:
1264 if common.verify_old_apk_signature(apkfile):
1265 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1273 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1274 move_apk_between_sections(repodir, 'archive', apk)
1276 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1277 return True, None, False
1279 apkzip = zipfile.ZipFile(apkfile, 'r')
1281 # if an APK has files newer than the system time, suggest updating
1282 # the system clock. This is useful for offline systems, used for
1283 # signing, which do not have another source of clock sync info. It
1284 # has to be more than 24 hours newer because ZIP/APK files do not
1285 # store timezone info
1286 manifest = apkzip.getinfo('AndroidManifest.xml')
1287 if manifest.date_time[1] == 0: # month can't be zero
1288 logging.debug('AndroidManifest.xml has no date')
1290 dt_obj = datetime(*manifest.date_time)
1291 checkdt = dt_obj - timedelta(1)
1292 if datetime.today() < checkdt:
1293 logging.warning('System clock is older than manifest in: '
1295 + '\nSet clock to that time using:\n'
1296 + 'sudo date -s "' + str(dt_obj) + '"')
1298 # extract icons from APK zip file
1299 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1301 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1303 apkzip.close() # ensure that APK zip file gets closed
1305 # resize existing icons for densities missing in the APK
1306 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1308 if use_date_from_apk and manifest.date_time[1] != 0:
1309 default_date_param = datetime(*manifest.date_time)
1311 default_date_param = None
1313 # Record in known apks, getting the added date at the same time..
1314 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1315 default_date=default_date_param)
1317 apk['added'] = added
1319 apkcache[apkfilename] = apk
1322 return False, apk, cachechanged
1325 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1326 """Processes the apks in the given repo directory.
1328 This also extracts the icons.
1330 :param apkcache: current apk cache information
1331 :param repodir: repo directory to scan
1332 :param knownapks: known apks info
1333 :param use_date_from_apk: use date from APK (instead of current date)
1334 for newly added APKs
1335 :returns: (apks, cachechanged) where apks is a list of apk information,
1336 and cachechanged is True if the apkcache got changed.
1339 cachechanged = False
1341 for icon_dir in get_all_icon_dirs(repodir):
1342 if os.path.exists(icon_dir):
1344 shutil.rmtree(icon_dir)
1345 os.makedirs(icon_dir)
1347 os.makedirs(icon_dir)
1350 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1351 apkfilename = apkfile[len(repodir) + 1:]
1352 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1353 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1354 use_date_from_apk, ada, True)
1358 cachechanged = cachechanged or cachethis
1360 return apks, cachechanged
1363 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1365 Extracts icons from the given APK zip in various densities,
1366 saves them into given repo directory
1367 and stores their names in the APK metadata dictionary.
1369 :param icon_filename: A string representing the icon's file name
1370 :param apk: A populated dictionary containing APK metadata.
1371 Needs to have 'icons_src' key
1372 :param apkzip: An opened zipfile.ZipFile of the APK file
1373 :param repo_dir: The directory of the APK's repository
1374 :return: A list of icon densities that are missing
1376 empty_densities = []
1377 for density in screen_densities:
1378 if density not in apk['icons_src']:
1379 empty_densities.append(density)
1381 icon_src = apk['icons_src'][density]
1382 icon_dir = get_icon_dir(repo_dir, density)
1383 icon_dest = os.path.join(icon_dir, icon_filename)
1385 # Extract the icon files per density
1386 if icon_src.endswith('.xml'):
1387 png = os.path.basename(icon_src)[:-4] + '.png'
1388 for f in apkzip.namelist():
1390 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1391 if m and screen_resolutions[m.group(2)] == density:
1393 if icon_src.endswith('.xml'):
1394 empty_densities.append(density)
1397 with open(icon_dest, 'wb') as f:
1398 f.write(get_icon_bytes(apkzip, icon_src))
1399 apk['icons'][density] = icon_filename
1400 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1401 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1402 del apk['icons_src'][density]
1403 empty_densities.append(density)
1405 if '-1' in apk['icons_src']:
1406 icon_src = apk['icons_src']['-1']
1407 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1408 with open(icon_path, 'wb') as f:
1409 f.write(get_icon_bytes(apkzip, icon_src))
1411 im = Image.open(icon_path)
1412 dpi = px_to_dpi(im.size[0])
1413 for density in screen_densities:
1414 if density in apk['icons']:
1416 if density == screen_densities[-1] or dpi >= int(density):
1417 apk['icons'][density] = icon_filename
1418 shutil.move(icon_path,
1419 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1420 empty_densities.remove(density)
1422 except Exception as e:
1423 logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1426 apk['icon'] = icon_filename
1428 return empty_densities
1431 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1433 Resize existing icons for densities missing in the APK to ensure all densities are available
1435 :param empty_densities: A list of icon densities that are missing
1436 :param icon_filename: A string representing the icon's file name
1437 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1438 :param repo_dir: The directory of the APK's repository
1440 # First try resizing down to not lose quality
1442 for density in screen_densities:
1443 if density not in empty_densities:
1444 last_density = density
1446 if last_density is None:
1448 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1450 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1451 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1454 fp = open(last_icon_path, 'rb')
1457 size = dpi_to_px(density)
1459 im.thumbnail((size, size), Image.ANTIALIAS)
1460 im.save(icon_path, "PNG")
1461 empty_densities.remove(density)
1462 except Exception as e:
1463 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1468 # Then just copy from the highest resolution available
1470 for density in reversed(screen_densities):
1471 if density not in empty_densities:
1472 last_density = density
1475 if last_density is None:
1479 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1480 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1482 empty_densities.remove(density)
1484 for density in screen_densities:
1485 icon_dir = get_icon_dir(repo_dir, density)
1486 icon_dest = os.path.join(icon_dir, icon_filename)
1487 resize_icon(icon_dest, density)
1489 # Copy from icons-mdpi to icons since mdpi is the baseline density
1490 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1491 if os.path.isfile(baseline):
1492 apk['icons']['0'] = icon_filename
1493 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1496 def apply_info_from_latest_apk(apps, apks):
1498 Some information from the apks needs to be applied up to the application level.
1499 When doing this, we use the info from the most recent version's apk.
1500 We deal with figuring out when the app was added and last updated at the same time.
1502 for appid, app in apps.items():
1503 bestver = UNSET_VERSION_CODE
1505 if apk['packageName'] == appid:
1506 if apk['versionCode'] > bestver:
1507 bestver = apk['versionCode']
1511 if not app.added or apk['added'] < app.added:
1512 app.added = apk['added']
1513 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1514 app.lastUpdated = apk['added']
1517 logging.debug("Don't know when " + appid + " was added")
1518 if not app.lastUpdated:
1519 logging.debug("Don't know when " + appid + " was last updated")
1521 if bestver == UNSET_VERSION_CODE:
1523 if app.Name is None:
1524 app.Name = app.AutoName or appid
1526 logging.debug("Application " + appid + " has no packages")
1528 if app.Name is None:
1529 app.Name = bestapk['name']
1530 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1531 if app.CurrentVersionCode is None:
1532 app.CurrentVersionCode = str(bestver)
1535 def make_categories_txt(repodir, categories):
1536 '''Write a category list in the repo to allow quick access'''
1538 for cat in sorted(categories):
1539 catdata += cat + '\n'
1540 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1544 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1546 def filter_apk_list_sorted(apk_list):
1548 for apk in apk_list:
1549 if apk['packageName'] == appid:
1552 # Sort the apk list by version code. First is highest/newest.
1553 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1555 for appid, app in apps.items():
1557 if app.ArchivePolicy:
1558 keepversions = int(app.ArchivePolicy[:-9])
1560 keepversions = defaultkeepversions
1562 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1563 .format(appid, len(apks), keepversions, len(archapks)))
1565 current_app_apks = filter_apk_list_sorted(apks)
1566 if len(current_app_apks) > keepversions:
1567 # Move back the ones we don't want.
1568 for apk in current_app_apks[keepversions:]:
1569 move_apk_between_sections(repodir, archivedir, apk)
1570 archapks.append(apk)
1573 current_app_archapks = filter_apk_list_sorted(archapks)
1574 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1576 # Move forward the ones we want again, except DisableAlgorithm
1577 for apk in current_app_archapks:
1578 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1579 move_apk_between_sections(archivedir, repodir, apk)
1580 archapks.remove(apk)
1583 if kept == keepversions:
1587 def move_apk_between_sections(from_dir, to_dir, apk):
1588 """move an APK from repo to archive or vice versa"""
1590 def _move_file(from_dir, to_dir, filename, ignore_missing):
1591 from_path = os.path.join(from_dir, filename)
1592 if ignore_missing and not os.path.exists(from_path):
1594 to_path = os.path.join(to_dir, filename)
1595 if not os.path.exists(to_dir):
1597 shutil.move(from_path, to_path)
1599 if from_dir == to_dir:
1602 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1603 _move_file(from_dir, to_dir, apk['apkName'], False)
1604 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1605 for density in all_screen_densities:
1606 from_icon_dir = get_icon_dir(from_dir, density)
1607 to_icon_dir = get_icon_dir(to_dir, density)
1608 if density not in apk['icons']:
1610 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1611 if 'srcname' in apk:
1612 _move_file(from_dir, to_dir, apk['srcname'], False)
1615 def add_apks_to_per_app_repos(repodir, apks):
1616 apks_per_app = dict()
1618 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1619 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1620 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1621 apks_per_app[apk['packageName']] = apk
1623 if not os.path.exists(apk['per_app_icons']):
1624 logging.info('Adding new repo for only ' + apk['packageName'])
1625 os.makedirs(apk['per_app_icons'])
1627 apkpath = os.path.join(repodir, apk['apkName'])
1628 shutil.copy(apkpath, apk['per_app_repo'])
1629 apksigpath = apkpath + '.sig'
1630 if os.path.exists(apksigpath):
1631 shutil.copy(apksigpath, apk['per_app_repo'])
1632 apkascpath = apkpath + '.asc'
1633 if os.path.exists(apkascpath):
1634 shutil.copy(apkascpath, apk['per_app_repo'])
1637 def create_metadata_from_template(apk):
1638 '''create a new metadata file using internal or external template
1640 Generate warnings for apk's with no metadata (or create skeleton
1641 metadata files, if requested on the command line). Though the
1642 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1643 since those impose things on the metadata file made from the
1644 template: field sort order, empty field value, formatting, etc.
1648 if os.path.exists('template.yml'):
1649 with open('template.yml') as f:
1651 if 'name' in apk and apk['name'] != '':
1652 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1653 r'\1 ' + apk['name'],
1655 flags=re.IGNORECASE | re.MULTILINE)
1657 logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1658 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1659 r'\1 ' + apk['packageName'],
1661 flags=re.IGNORECASE | re.MULTILINE)
1662 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1666 app['Categories'] = [os.path.basename(os.getcwd())]
1667 # include some blanks as part of the template
1668 app['AuthorName'] = ''
1671 app['IssueTracker'] = ''
1672 app['SourceCode'] = ''
1673 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1674 if 'name' in apk and apk['name'] != '':
1675 app['Name'] = apk['name']
1677 logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1678 app['Name'] = apk['packageName']
1679 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1680 yaml.dump(app, f, default_flow_style=False)
1681 logging.info("Generated skeleton metadata for " + apk['packageName'])
1690 global config, options
1692 # Parse command line...
1693 parser = ArgumentParser()
1694 common.setup_global_opts(parser)
1695 parser.add_argument("--create-key", action="store_true", default=False,
1696 help=_("Create a repo signing key in a keystore"))
1697 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1698 help=_("Create skeleton metadata files that are missing"))
1699 parser.add_argument("--delete-unknown", action="store_true", default=False,
1700 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1701 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1702 help=_("Report on build data status"))
1703 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1704 help=_("Interactively ask about things that need updating."))
1705 parser.add_argument("-I", "--icons", action="store_true", default=False,
1706 help=_("Resize all the icons exceeding the max pixel size and exit"))
1707 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1708 help=_("Specify editor to use in interactive mode. Default ") +
1709 "is /etc/alternatives/editor")
1710 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1711 help=_("Update the wiki"))
1712 parser.add_argument("--pretty", action="store_true", default=False,
1713 help=_("Produce human-readable index.xml"))
1714 parser.add_argument("--clean", action="store_true", default=False,
1715 help=_("Clean update - don't uses caches, reprocess all APKs"))
1716 parser.add_argument("--nosign", action="store_true", default=False,
1717 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1718 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1719 help=_("Use date from APK instead of current time for newly added APKs"))
1720 parser.add_argument("--rename-apks", action="store_true", default=False,
1721 help=_("Rename APK files that do not match package.name_123.apk"))
1722 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1723 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1724 metadata.add_metadata_arguments(parser)
1725 options = parser.parse_args()
1726 metadata.warnings_action = options.W
1728 config = common.read_config(options)
1730 if not ('jarsigner' in config and 'keytool' in config):
1731 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1734 if config['archive_older'] != 0:
1735 repodirs.append('archive')
1736 if not os.path.exists('archive'):
1740 resize_all_icons(repodirs)
1743 if options.rename_apks:
1744 options.clean = True
1746 # check that icons exist now, rather than fail at the end of `fdroid update`
1747 for k in ['repo_icon', 'archive_icon']:
1749 if not os.path.exists(config[k]):
1750 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1753 # if the user asks to create a keystore, do it now, reusing whatever it can
1754 if options.create_key:
1755 if os.path.exists(config['keystore']):
1756 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1757 logging.critical("\t'" + config['keystore'] + "'")
1760 if 'repo_keyalias' not in config:
1761 config['repo_keyalias'] = socket.getfqdn()
1762 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1763 if 'keydname' not in config:
1764 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1765 common.write_to_config(config, 'keydname', config['keydname'])
1766 if 'keystore' not in config:
1767 config['keystore'] = common.default_config['keystore']
1768 common.write_to_config(config, 'keystore', config['keystore'])
1770 password = common.genpassword()
1771 if 'keystorepass' not in config:
1772 config['keystorepass'] = password
1773 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1774 if 'keypass' not in config:
1775 config['keypass'] = password
1776 common.write_to_config(config, 'keypass', config['keypass'])
1777 common.genkeystore(config)
1780 apps = metadata.read_metadata()
1782 # Generate a list of categories...
1784 for app in apps.values():
1785 categories.update(app.Categories)
1787 # Read known apks data (will be updated and written back when we've finished)
1788 knownapks = common.KnownApks()
1791 apkcache = get_cache()
1793 # Delete builds for disabled apps
1794 delete_disabled_builds(apps, apkcache, repodirs)
1796 # Scan all apks in the main repo
1797 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1799 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1800 options.use_date_from_apk)
1801 cachechanged = cachechanged or fcachechanged
1804 if apk['packageName'] not in apps:
1805 if options.create_metadata:
1806 create_metadata_from_template(apk)
1807 apps = metadata.read_metadata()
1809 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1810 if options.delete_unknown:
1811 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1812 rmf = os.path.join(repodirs[0], apk['apkName'])
1813 if not os.path.exists(rmf):
1814 logging.error("Could not find {0} to remove it".format(rmf))
1818 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1820 copy_triple_t_store_metadata(apps)
1821 insert_obbs(repodirs[0], apps, apks)
1822 insert_localized_app_metadata(apps)
1823 translate_per_build_anti_features(apps, apks)
1825 # Scan the archive repo for apks as well
1826 if len(repodirs) > 1:
1827 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1833 # Apply information from latest apks to the application and update dates
1834 apply_info_from_latest_apk(apps, apks + archapks)
1836 # Sort the app list by name, then the web site doesn't have to by default.
1837 # (we had to wait until we'd scanned the apks to do this, because mostly the
1838 # name comes from there!)
1839 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1841 # APKs are placed into multiple repos based on the app package, providing
1842 # per-app subscription feeds for nightly builds and things like it
1843 if config['per_app_repos']:
1844 add_apks_to_per_app_repos(repodirs[0], apks)
1845 for appid, app in apps.items():
1846 repodir = os.path.join(appid, 'fdroid', 'repo')
1848 appdict[appid] = app
1849 if os.path.isdir(repodir):
1850 index.make(appdict, [appid], apks, repodir, False)
1852 logging.info('Skipping index generation for ' + appid)
1855 if len(repodirs) > 1:
1856 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1858 # Make the index for the main repo...
1859 index.make(apps, sortedids, apks, repodirs[0], False)
1860 make_categories_txt(repodirs[0], categories)
1862 # If there's an archive repo, make the index for it. We already scanned it
1864 if len(repodirs) > 1:
1865 index.make(apps, sortedids, archapks, repodirs[1], True)
1867 git_remote = config.get('binary_transparency_remote')
1868 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1870 btlog.make_binary_transparency_log(repodirs)
1872 if config['update_stats']:
1873 # Update known apks info...
1874 knownapks.writeifchanged()
1876 # Generate latest apps data for widget
1877 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1879 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1881 appid = line.rstrip()
1882 data += appid + "\t"
1884 data += app.Name + "\t"
1885 if app.icon is not None:
1886 data += app.icon + "\t"
1887 data += app.License + "\n"
1888 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1892 write_cache(apkcache)
1894 # Update the wiki...
1896 update_wiki(apps, sortedids, apks + archapks)
1898 logging.info(_("Finished"))
1901 if __name__ == "__main__":