3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
38 from PIL import Image, PngImagePlugin
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
90 def dpi_to_px(density):
91 return (int(density) * 48) / 160
95 return (int(px) * 160) / 48
98 def get_icon_dir(repodir, density):
100 return os.path.join(repodir, "icons")
101 return os.path.join(repodir, "icons-%s" % density)
104 def get_icon_dirs(repodir):
105 for density in screen_densities:
106 yield get_icon_dir(repodir, density)
109 def get_all_icon_dirs(repodir):
110 for density in all_screen_densities:
111 yield get_icon_dir(repodir, density)
114 def update_wiki(apps, sortedids, apks):
117 :param apps: fully populated list of all applications
118 :param apks: all apks, except...
120 logging.info("Updating wiki")
122 wikiredircat = 'App Redirects'
124 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
125 path=config['wiki_path'])
126 site.login(config['wiki_user'], config['wiki_password'])
128 generated_redirects = {}
130 for appid in sortedids:
131 app = metadata.App(apps[appid])
135 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
137 for af in sorted(app.AntiFeatures):
138 wikidata += '{{AntiFeature|' + af + '}}\n'
143 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
146 app.added.strftime('%Y-%m-%d') if app.added else '',
147 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
163 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
165 wikidata += app.Summary
166 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
168 wikidata += "=Description=\n"
169 wikidata += metadata.description_wiki(app.Description) + "\n"
171 wikidata += "=Maintainer Notes=\n"
172 if app.MaintainerNotes:
173 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
174 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
176 # Get a list of all packages for this application...
178 gotcurrentver = False
182 if apk['packageName'] == appid:
183 if str(apk['versionCode']) == app.CurrentVersionCode:
186 # Include ones we can't build, as a special case...
187 for build in app.builds:
189 if build.versionCode == app.CurrentVersionCode:
191 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
192 apklist.append({'versionCode': int(build.versionCode),
193 'versionName': build.versionName,
194 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
199 if apk['versionCode'] == int(build.versionCode):
204 apklist.append({'versionCode': int(build.versionCode),
205 'versionName': build.versionName,
206 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
208 if app.CurrentVersionCode == '0':
210 # Sort with most recent first...
211 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
213 wikidata += "=Versions=\n"
214 if len(apklist) == 0:
215 wikidata += "We currently have no versions of this app available."
216 elif not gotcurrentver:
217 wikidata += "We don't have the current version of this app."
219 wikidata += "We have the current version of this app."
220 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
221 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
222 if len(app.NoSourceSince) > 0:
223 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
224 if len(app.CurrentVersion) > 0:
225 wikidata += "The current (recommended) version is " + app.CurrentVersion
226 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
229 wikidata += "==" + apk['versionName'] + "==\n"
231 if 'buildproblem' in apk:
232 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
235 wikidata += "This version is built and signed by "
237 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
239 wikidata += "the original developer.\n\n"
240 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
242 wikidata += '\n[[Category:' + wikicat + ']]\n'
243 if len(app.NoSourceSince) > 0:
244 wikidata += '\n[[Category:Apps missing source code]]\n'
245 if validapks == 0 and not app.Disabled:
246 wikidata += '\n[[Category:Apps with no packages]]\n'
247 if cantupdate and not app.Disabled:
248 wikidata += "\n[[Category:Apps we cannot update]]\n"
249 if buildfails and not app.Disabled:
250 wikidata += "\n[[Category:Apps with failing builds]]\n"
251 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
252 wikidata += '\n[[Category:Apps to Update]]\n'
254 wikidata += '\n[[Category:Apps that are disabled]]\n'
255 if app.UpdateCheckMode == 'None' and not app.Disabled:
256 wikidata += '\n[[Category:Apps with no update check]]\n'
257 for appcat in app.Categories:
258 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
260 # We can't have underscores in the page name, even if they're in
261 # the package ID, because MediaWiki messes with them...
262 pagename = appid.replace('_', ' ')
264 # Drop a trailing newline, because mediawiki is going to drop it anyway
265 # and it we don't we'll think the page has changed when it hasn't...
266 if wikidata.endswith('\n'):
267 wikidata = wikidata[:-1]
269 generated_pages[pagename] = wikidata
271 # Make a redirect from the name to the ID too, unless there's
272 # already an existing page with the name and it isn't a redirect.
274 apppagename = app.Name.replace('_', ' ')
275 apppagename = apppagename.replace('{', '')
276 apppagename = apppagename.replace('}', ' ')
277 apppagename = apppagename.replace(':', ' ')
278 apppagename = apppagename.replace('[', ' ')
279 apppagename = apppagename.replace(']', ' ')
280 # Drop double spaces caused mostly by replacing ':' above
281 apppagename = apppagename.replace(' ', ' ')
282 for expagename in site.allpages(prefix=apppagename,
283 filterredir='nonredirects',
285 if expagename == apppagename:
287 # Another reason not to make the redirect page is if the app name
288 # is the same as it's ID, because that will overwrite the real page
289 # with an redirect to itself! (Although it seems like an odd
290 # scenario this happens a lot, e.g. where there is metadata but no
291 # builds or binaries to extract a name from.
292 if apppagename == pagename:
295 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
297 for tcat, genp in [(wikicat, generated_pages),
298 (wikiredircat, generated_redirects)]:
299 catpages = site.Pages['Category:' + tcat]
301 for page in catpages:
302 existingpages.append(page.name)
303 if page.name in genp:
304 pagetxt = page.edit()
305 if pagetxt != genp[page.name]:
306 logging.debug("Updating modified page " + page.name)
307 page.save(genp[page.name], summary='Auto-updated')
309 logging.debug("Page " + page.name + " is unchanged")
311 logging.warn("Deleting page " + page.name)
312 page.delete('No longer published')
313 for pagename, text in genp.items():
314 logging.debug("Checking " + pagename)
315 if pagename not in existingpages:
316 logging.debug("Creating page " + pagename)
318 newpage = site.Pages[pagename]
319 newpage.save(text, summary='Auto-created')
320 except Exception as e:
321 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
323 # Purge server cache to ensure counts are up to date
324 site.pages['Repository Maintenance'].purge()
327 def delete_disabled_builds(apps, apkcache, repodirs):
328 """Delete disabled build outputs.
330 :param apps: list of all applications, as per metadata.read_metadata
331 :param apkcache: current apk cache information
332 :param repodirs: the repo directories to process
334 for appid, app in apps.items():
335 for build in app['builds']:
336 if not build.disable:
338 apkfilename = common.get_release_filename(app, build)
339 iconfilename = "%s.%s.png" % (
342 for repodir in repodirs:
344 os.path.join(repodir, apkfilename),
345 os.path.join(repodir, apkfilename + '.asc'),
346 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
348 for density in all_screen_densities:
349 repo_dir = get_icon_dir(repodir, density)
350 files.append(os.path.join(repo_dir, iconfilename))
353 if os.path.exists(f):
354 logging.info("Deleting disabled build output " + f)
356 if apkfilename in apkcache:
357 del apkcache[apkfilename]
360 def resize_icon(iconpath, density):
362 if not os.path.isfile(iconpath):
367 fp = open(iconpath, 'rb')
369 size = dpi_to_px(density)
371 if any(length > size for length in im.size):
373 im.thumbnail((size, size), Image.ANTIALIAS)
374 logging.debug("%s was too large at %s - new size is %s" % (
375 iconpath, oldsize, im.size))
376 im.save(iconpath, "PNG", optimize=True,
377 pnginfo=BLANK_PNG_INFO, icc_profile=None)
379 except Exception as e:
380 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
387 def resize_all_icons(repodirs):
388 """Resize all icons that exceed the max size
390 :param repodirs: the repo directories to process
392 for repodir in repodirs:
393 for density in screen_densities:
394 icon_dir = get_icon_dir(repodir, density)
395 icon_glob = os.path.join(icon_dir, '*.png')
396 for iconpath in glob.glob(icon_glob):
397 resize_icon(iconpath, density)
401 """ Get the signing certificate of an apk. To get the same md5 has that
402 Android gets, we encode the .RSA certificate in a specific format and pass
403 it hex-encoded to the md5 digest algorithm.
405 :param apkpath: path to the apk
406 :returns: A string containing the md5 of the signature of the apk or None
407 if an error occurred.
410 with zipfile.ZipFile(apkpath, 'r') as apk:
411 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
414 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
417 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
420 cert = apk.read(certs[0])
422 cert_encoded = common.get_certificate(cert)
424 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
427 def get_cache_file():
428 return os.path.join('tmp', 'apkcache')
432 """Get the cached dict of the APK index
434 Gather information about all the apk files in the repo directory,
435 using cached data if possible. Some of the index operations take a
436 long time, like calculating the SHA-256 and verifying the APK
439 The cache is invalidated if the metadata version is different, or
440 the 'allow_disabled_algorithms' config/option is different. In
441 those cases, there is no easy way to know what has changed from
442 the cache, so just rerun the whole thing.
447 apkcachefile = get_cache_file()
448 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
449 if not options.clean and os.path.exists(apkcachefile):
450 with open(apkcachefile, 'rb') as cf:
451 apkcache = pickle.load(cf, encoding='utf-8')
452 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
453 or apkcache.get('allow_disabled_algorithms') != ada:
458 apkcache["METADATA_VERSION"] = METADATA_VERSION
459 apkcache['allow_disabled_algorithms'] = ada
464 def write_cache(apkcache):
465 apkcachefile = get_cache_file()
466 cache_path = os.path.dirname(apkcachefile)
467 if not os.path.exists(cache_path):
468 os.makedirs(cache_path)
469 with open(apkcachefile, 'wb') as cf:
470 pickle.dump(apkcache, cf)
473 def get_icon_bytes(apkzip, iconsrc):
474 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
476 return apkzip.read(iconsrc)
478 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
481 def sha256sum(filename):
482 '''Calculate the sha256 of the given file'''
483 sha = hashlib.sha256()
484 with open(filename, 'rb') as f:
490 return sha.hexdigest()
493 def has_known_vulnerability(filename):
494 """checks for known vulnerabilities in the APK
496 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
497 version. Google also enforces this:
498 https://support.google.com/faqs/answer/6376725?hl=en
500 Checks whether there are more than one classes.dex or AndroidManifest.xml
501 files, which is invalid and an essential part of the "Master Key" attack.
502 http://www.saurik.com/id/17
504 Janus is similar to Master Key but is perhaps easier to scan for.
505 https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
510 # statically load this pattern
511 if not hasattr(has_known_vulnerability, "pattern"):
512 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
514 with open(filename.encode(), 'rb') as fp:
516 if first4 != b'\x50\x4b\x03\x04':
517 raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
518 .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
519 + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
522 with zipfile.ZipFile(filename) as zf:
523 for name in zf.namelist():
524 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
527 chunk = lib.read(4096)
530 m = has_known_vulnerability.pattern.search(chunk)
532 version = m.group(1).decode('ascii')
533 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
534 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
535 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
536 logging.debug(_('"{path}" contains recent {name} ({version})')
537 .format(path=filename, name=name, version=version))
539 logging.warning(_('"{path}" contains outdated {name} ({version})')
540 .format(path=filename, name=name, version=version))
543 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
544 if name in files_in_apk:
545 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
546 .format(apkfilename=filename, name=name))
548 files_in_apk.add(name)
552 def insert_obbs(repodir, apps, apks):
553 """Scans the .obb files in a given repo directory and adds them to the
554 relevant APK instances. OBB files have versionCodes like APK
555 files, and they are loosely associated. If there is an OBB file
556 present, then any APK with the same or higher versionCode will use
557 that OBB file. There are two OBB types: main and patch, each APK
558 can only have only have one of each.
560 https://developer.android.com/google/play/expansion-files.html
562 :param repodir: repo directory to scan
563 :param apps: list of current, valid apps
564 :param apks: current information on all APKs
568 def obbWarnDelete(f, msg):
569 logging.warning(msg + ' ' + f)
570 if options.delete_unknown:
571 logging.error(_("Deleting unknown file: {path}").format(path=f))
575 java_Integer_MIN_VALUE = -pow(2, 31)
576 currentPackageNames = apps.keys()
577 for f in glob.glob(os.path.join(repodir, '*.obb')):
578 obbfile = os.path.basename(f)
579 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
580 chunks = obbfile.split('.')
581 if chunks[0] != 'main' and chunks[0] != 'patch':
582 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
584 if not re.match(r'^-?[0-9]+$', chunks[1]):
585 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
586 .format(name=chunks[0]))
588 versionCode = int(chunks[1])
589 packagename = ".".join(chunks[2:-1])
591 highestVersionCode = java_Integer_MIN_VALUE
592 if packagename not in currentPackageNames:
593 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
596 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
597 highestVersionCode = apk['versionCode']
598 if versionCode > highestVersionCode:
599 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
600 .format(integer=str(versionCode)))
602 obbsha256 = sha256sum(f)
603 obbs.append((packagename, versionCode, obbfile, obbsha256))
606 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
607 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
608 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
609 apk['obbMainFile'] = obbfile
610 apk['obbMainFileSha256'] = obbsha256
611 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
612 apk['obbPatchFile'] = obbfile
613 apk['obbPatchFileSha256'] = obbsha256
614 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
618 def translate_per_build_anti_features(apps, apks):
619 """Grab the anti-features list from the build metadata
621 For most Anti-Features, they are really most applicable per-APK,
622 not for an app. An app can fix a vulnerability, add/remove
623 tracking, etc. This reads the 'antifeatures' list from the Build
624 entries in the fdroiddata metadata file, then transforms it into
625 the 'antiFeatures' list of unique items for the index.
627 The field key is all lower case in the metadata file to match the
628 rest of the Build fields. It is 'antiFeatures' camel case in the
629 implementation, index, and fdroidclient since it is translated
630 from the build 'antifeatures' field, not directly included.
634 antiFeatures = dict()
635 for packageName, app in apps.items():
637 for build in app['builds']:
638 afl = build.get('antifeatures')
640 d[int(build.versionCode)] = afl
642 antiFeatures[packageName] = d
645 d = antiFeatures.get(apk['packageName'])
647 afl = d.get(apk['versionCode'])
649 apk['antiFeatures'].update(afl)
652 def _get_localized_dict(app, locale):
653 '''get the dict to add localized store metadata to'''
654 if 'localized' not in app:
655 app['localized'] = collections.OrderedDict()
656 if locale not in app['localized']:
657 app['localized'][locale] = collections.OrderedDict()
658 return app['localized'][locale]
661 def _set_localized_text_entry(app, locale, key, f):
662 limit = config['char_limits'][key]
663 localized = _get_localized_dict(app, locale)
665 text = fp.read()[:limit]
667 localized[key] = text
670 def _set_author_entry(app, key, f):
671 limit = config['char_limits']['author']
673 text = fp.read()[:limit]
678 def _strip_and_copy_image(inpath, outpath):
679 """Remove any metadata from image and copy it to new path
681 Sadly, image metadata like EXIF can be used to exploit devices.
682 It is not used at all in the F-Droid ecosystem, so its much safer
683 just to remove it entirely.
687 extension = common.get_extension(inpath)[1]
688 if os.path.isdir(outpath):
689 outpath = os.path.join(outpath, os.path.basename(inpath))
690 if extension == 'png':
691 with open(inpath, 'rb') as fp:
692 in_image = Image.open(fp)
693 in_image.save(outpath, "PNG", optimize=True,
694 pnginfo=BLANK_PNG_INFO, icc_profile=None)
695 elif extension == 'jpg' or extension == 'jpeg':
696 with open(inpath, 'rb') as fp:
697 in_image = Image.open(fp)
698 data = list(in_image.getdata())
699 out_image = Image.new(in_image.mode, in_image.size)
700 out_image.putdata(data)
701 out_image.save(outpath, "JPEG", optimize=True)
703 raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
704 .format(extension=extension))
707 def copy_triple_t_store_metadata(apps):
708 """Include store metadata from the app's source repo
710 The Triple-T Gradle Play Publisher is a plugin that has a standard
711 file layout for all of the metadata and graphics that the Google
712 Play Store accepts. Since F-Droid has the git repo, it can just
713 pluck those files directly. This method reads any text files into
714 the app dict, then copies any graphics into the fdroid repo
717 This needs to be run before insert_localized_app_metadata() so that
718 the graphics files that are copied into the fdroid repo get
721 https://github.com/Triple-T/gradle-play-publisher#upload-images
722 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
726 if not os.path.isdir('build'):
727 return # nothing to do
729 for packageName, app in apps.items():
730 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
731 logging.debug('Triple-T Gradle Play Publisher: ' + d)
732 for root, dirs, files in os.walk(d):
733 segments = root.split('/')
734 locale = segments[-2]
736 if f == 'fulldescription':
737 _set_localized_text_entry(app, locale, 'description',
738 os.path.join(root, f))
740 elif f == 'shortdescription':
741 _set_localized_text_entry(app, locale, 'summary',
742 os.path.join(root, f))
745 _set_localized_text_entry(app, locale, 'name',
746 os.path.join(root, f))
749 _set_localized_text_entry(app, locale, 'video',
750 os.path.join(root, f))
752 elif f == 'whatsnew':
753 _set_localized_text_entry(app, segments[-1], 'whatsNew',
754 os.path.join(root, f))
756 elif f == 'contactEmail':
757 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
759 elif f == 'contactPhone':
760 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
762 elif f == 'contactWebsite':
763 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
766 base, extension = common.get_extension(f)
767 dirname = os.path.basename(root)
768 if extension in ALLOWED_EXTENSIONS \
769 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
770 if segments[-2] == 'listing':
771 locale = segments[-3]
773 locale = segments[-2]
774 destdir = os.path.join('repo', packageName, locale, dirname)
775 os.makedirs(destdir, mode=0o755, exist_ok=True)
776 sourcefile = os.path.join(root, f)
777 destfile = os.path.join(destdir, os.path.basename(f))
778 logging.debug('copying ' + sourcefile + ' ' + destfile)
779 _strip_and_copy_image(sourcefile, destfile)
782 def insert_localized_app_metadata(apps):
783 """scans standard locations for graphics and localized text
785 Scans for localized description files, store graphics, and
786 screenshot PNG files in statically defined screenshots directory
787 and adds them to the app metadata. The screenshots and graphic
788 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
789 and must be in the following layout:
790 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
792 repo/packageName/locale/featureGraphic.png
793 repo/packageName/locale/phoneScreenshots/1.png
794 repo/packageName/locale/phoneScreenshots/2.png
796 The changelog files must be text files named with the versionCode
797 ending with ".txt" and must be in the following layout:
798 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
800 repo/packageName/locale/changelogs/12345.txt
802 This will scan the each app's source repo then the metadata/ dir
803 for these standard locations of changelog files. If it finds
804 them, they will be added to the dict of all packages, with the
805 versions in the metadata/ folder taking precendence over the what
806 is in the app's source repo.
808 Where "packageName" is the app's packageName and "locale" is the locale
809 of the graphics, e.g. what language they are in, using the IETF RFC5646
810 format (en-US, fr-CA, es-MX, etc).
812 This will also scan the app's git for a fastlane folder, and the
813 metadata/ folder and the apps' source repos for standard locations
814 of graphic and screenshot files. If it finds them, it will copy
815 them into the repo. The fastlane files follow this pattern:
816 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
820 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
821 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
822 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
823 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
825 for srcd in sorted(sourcedirs):
826 if not os.path.isdir(srcd):
828 for root, dirs, files in os.walk(srcd):
829 segments = root.split('/')
830 packageName = segments[1]
831 if packageName not in apps:
832 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
834 locale = segments[-1]
835 destdir = os.path.join('repo', packageName, locale)
837 # flavours specified in build receipt
839 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
840 and 'gradle' in apps[packageName].builds[-1]:
841 build_flavours = apps[packageName].builds[-1].gradle
843 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
844 logging.debug("ignoring due to wrong flavour")
848 if f in ('description.txt', 'full_description.txt'):
849 _set_localized_text_entry(apps[packageName], locale, 'description',
850 os.path.join(root, f))
852 elif f in ('summary.txt', 'short_description.txt'):
853 _set_localized_text_entry(apps[packageName], locale, 'summary',
854 os.path.join(root, f))
856 elif f in ('name.txt', 'title.txt'):
857 _set_localized_text_entry(apps[packageName], locale, 'name',
858 os.path.join(root, f))
860 elif f == 'video.txt':
861 _set_localized_text_entry(apps[packageName], locale, 'video',
862 os.path.join(root, f))
864 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
865 locale = segments[-2]
866 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
867 os.path.join(root, f))
870 base, extension = common.get_extension(f)
871 if locale == 'images':
872 locale = segments[-2]
873 destdir = os.path.join('repo', packageName, locale)
874 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
875 os.makedirs(destdir, mode=0o755, exist_ok=True)
876 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
877 _strip_and_copy_image(os.path.join(root, f), destdir)
879 if d in SCREENSHOT_DIRS:
880 if locale == 'images':
881 locale = segments[-2]
882 destdir = os.path.join('repo', packageName, locale)
883 for f in glob.glob(os.path.join(root, d, '*.*')):
884 _ignored, extension = common.get_extension(f)
885 if extension in ALLOWED_EXTENSIONS:
886 screenshotdestdir = os.path.join(destdir, d)
887 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
888 logging.debug('copying ' + f + ' ' + screenshotdestdir)
889 _strip_and_copy_image(f, screenshotdestdir)
891 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
893 if not os.path.isdir(d):
895 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
896 if not os.path.isfile(f):
898 segments = f.split('/')
899 packageName = segments[1]
901 screenshotdir = segments[3]
902 filename = os.path.basename(f)
903 base, extension = common.get_extension(filename)
905 if packageName not in apps:
906 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
907 .format(path=filename, name=packageName))
909 graphics = _get_localized_dict(apps[packageName], locale)
911 if extension not in ALLOWED_EXTENSIONS:
912 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
913 elif base in GRAPHIC_NAMES:
914 # there can only be zero or one of these per locale
915 graphics[base] = filename
916 elif screenshotdir in SCREENSHOT_DIRS:
917 # there can any number of these per locale
918 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
919 if screenshotdir not in graphics:
920 graphics[screenshotdir] = []
921 graphics[screenshotdir].append(filename)
923 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
926 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
927 """Scan a repo for all files with an extension except APK/OBB
929 :param apkcache: current cached info about all repo files
930 :param repodir: repo directory to scan
931 :param knownapks: list of all known files, as per metadata.read_metadata
932 :param use_date_from_file: use date from file (instead of current date)
933 for newly added files
938 repodir = repodir.encode('utf-8')
939 for name in os.listdir(repodir):
940 file_extension = common.get_file_extension(name)
941 if file_extension == 'apk' or file_extension == 'obb':
943 filename = os.path.join(repodir, name)
944 name_utf8 = name.decode('utf-8')
945 if filename.endswith(b'_src.tar.gz'):
946 logging.debug(_('skipping source tarball: {path}')
947 .format(path=filename.decode('utf-8')))
949 if not common.is_repo_file(filename):
951 stat = os.stat(filename)
952 if stat.st_size == 0:
953 raise FDroidException(_('{path} is zero size!')
954 .format(path=filename))
956 shasum = sha256sum(filename)
959 repo_file = apkcache[name]
960 # added time is cached as tuple but used here as datetime instance
961 if 'added' in repo_file:
962 a = repo_file['added']
963 if isinstance(a, datetime):
964 repo_file['added'] = a
966 repo_file['added'] = datetime(*a[:6])
967 if repo_file.get('hash') == shasum:
968 logging.debug(_("Reading {apkfilename} from cache")
969 .format(apkfilename=name_utf8))
972 logging.debug(_("Ignoring stale cache data for {apkfilename}")
973 .format(apkfilename=name_utf8))
976 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
977 repo_file = collections.OrderedDict()
978 repo_file['name'] = os.path.splitext(name_utf8)[0]
979 # TODO rename apkname globally to something more generic
980 repo_file['apkName'] = name_utf8
981 repo_file['hash'] = shasum
982 repo_file['hashType'] = 'sha256'
983 repo_file['versionCode'] = 0
984 repo_file['versionName'] = shasum
985 # the static ID is the SHA256 unless it is set in the metadata
986 repo_file['packageName'] = shasum
988 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
990 repo_file['packageName'] = m.group(1)
991 repo_file['versionCode'] = int(m.group(2))
992 srcfilename = name + b'_src.tar.gz'
993 if os.path.exists(os.path.join(repodir, srcfilename)):
994 repo_file['srcname'] = srcfilename.decode('utf-8')
995 repo_file['size'] = stat.st_size
997 apkcache[name] = repo_file
1000 if use_date_from_file:
1001 timestamp = stat.st_ctime
1002 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1004 default_date_param = None
1006 # Record in knownapks, getting the added date at the same time..
1007 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1008 default_date=default_date_param)
1010 repo_file['added'] = added
1012 repo_files.append(repo_file)
1014 return repo_files, cachechanged
1017 def scan_apk(apk_file):
1019 Scans an APK file and returns dictionary with metadata of the APK.
1021 Attention: This does *not* verify that the APK signature is correct.
1023 :param apk_file: The (ideally absolute) path to the APK file
1024 :raises BuildException
1025 :return A dict containing APK metadata
1028 'hash': sha256sum(apk_file),
1029 'hashType': 'sha256',
1030 'uses-permission': [],
1031 'uses-permission-sdk-23': [],
1035 'antiFeatures': set(),
1038 if SdkToolsPopen(['aapt', 'version'], output=False):
1039 scan_apk_aapt(apk, apk_file)
1041 scan_apk_androguard(apk, apk_file)
1043 # Get the signature, or rather the signing key fingerprints
1044 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1045 apk['sig'] = getsig(apk_file)
1047 raise BuildException("Failed to get apk signature")
1048 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1050 if not apk.get('signer'):
1051 raise BuildException("Failed to get apk signing key fingerprint")
1053 # Get size of the APK
1054 apk['size'] = os.path.getsize(apk_file)
1056 if 'minSdkVersion' not in apk:
1057 logging.warning("No SDK version information found in {0}".format(apk_file))
1058 apk['minSdkVersion'] = 1
1060 # Check for known vulnerabilities
1061 if has_known_vulnerability(apk_file):
1062 apk['antiFeatures'].add('KnownVuln')
1067 def scan_apk_aapt(apk, apkfile):
1068 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1069 if p.returncode != 0:
1070 if options.delete_unknown:
1071 if os.path.exists(apkfile):
1072 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1075 logging.error("Could not find {0} to remove it".format(apkfile))
1077 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1078 raise BuildException(_("Invalid APK"))
1079 for line in p.output.splitlines():
1080 if line.startswith("package:"):
1082 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1083 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1084 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1085 except Exception as e:
1086 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1087 elif line.startswith("application:"):
1088 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1089 # Keep path to non-dpi icon in case we need it
1090 match = re.match(APK_ICON_PAT_NODPI, line)
1092 apk['icons_src']['-1'] = match.group(1)
1093 elif line.startswith("launchable-activity:"):
1094 # Only use launchable-activity as fallback to application
1096 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1097 if '-1' not in apk['icons_src']:
1098 match = re.match(APK_ICON_PAT_NODPI, line)
1100 apk['icons_src']['-1'] = match.group(1)
1101 elif line.startswith("application-icon-"):
1102 match = re.match(APK_ICON_PAT, line)
1104 density = match.group(1)
1105 path = match.group(2)
1106 apk['icons_src'][density] = path
1107 elif line.startswith("sdkVersion:"):
1108 m = re.match(APK_SDK_VERSION_PAT, line)
1110 logging.error(line.replace('sdkVersion:', '')
1111 + ' is not a valid minSdkVersion!')
1113 apk['minSdkVersion'] = m.group(1)
1114 # if target not set, default to min
1115 if 'targetSdkVersion' not in apk:
1116 apk['targetSdkVersion'] = m.group(1)
1117 elif line.startswith("targetSdkVersion:"):
1118 m = re.match(APK_SDK_VERSION_PAT, line)
1120 logging.error(line.replace('targetSdkVersion:', '')
1121 + ' is not a valid targetSdkVersion!')
1123 apk['targetSdkVersion'] = m.group(1)
1124 elif line.startswith("maxSdkVersion:"):
1125 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1126 elif line.startswith("native-code:"):
1127 apk['nativecode'] = []
1128 for arch in line[13:].split(' '):
1129 apk['nativecode'].append(arch[1:-1])
1130 elif line.startswith('uses-permission:'):
1131 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1132 if perm_match['maxSdkVersion']:
1133 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1134 permission = UsesPermission(
1136 perm_match['maxSdkVersion']
1139 apk['uses-permission'].append(permission)
1140 elif line.startswith('uses-permission-sdk-23:'):
1141 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1142 if perm_match['maxSdkVersion']:
1143 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1144 permission_sdk_23 = UsesPermissionSdk23(
1146 perm_match['maxSdkVersion']
1149 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1151 elif line.startswith('uses-feature:'):
1152 feature = re.match(APK_FEATURE_PAT, line).group(1)
1153 # Filter out this, it's only added with the latest SDK tools and
1154 # causes problems for lots of apps.
1155 if feature != "android.hardware.screen.portrait" \
1156 and feature != "android.hardware.screen.landscape":
1157 if feature.startswith("android.feature."):
1158 feature = feature[16:]
1159 apk['features'].add(feature)
1162 def scan_apk_androguard(apk, apkfile):
1164 from androguard.core.bytecodes.apk import APK
1165 apkobject = APK(apkfile)
1166 if apkobject.is_valid_APK():
1167 arsc = apkobject.get_android_resources()
1169 if options.delete_unknown:
1170 if os.path.exists(apkfile):
1171 logging.error(_("Failed to get apk information, deleting {path}")
1172 .format(path=apkfile))
1175 logging.error(_("Could not find {path} to remove it")
1176 .format(path=apkfile))
1178 logging.error(_("Failed to get apk information, skipping {path}")
1179 .format(path=apkfile))
1180 raise BuildException(_("Invalid APK"))
1182 raise FDroidException("androguard library is not installed and aapt not present")
1183 except FileNotFoundError:
1184 logging.error(_("Could not open apk file for analysis"))
1185 raise BuildException(_("Invalid APK"))
1187 apk['packageName'] = apkobject.get_package()
1188 apk['versionCode'] = int(apkobject.get_androidversion_code())
1189 apk['versionName'] = apkobject.get_androidversion_name()
1190 if apk['versionName'][0] == "@":
1191 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1192 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1193 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1194 apk['name'] = apkobject.get_app_name()
1196 if apkobject.get_max_sdk_version() is not None:
1197 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1198 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1199 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1201 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1202 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1204 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1206 for file in apkobject.get_files():
1207 d_re = density_re.match(file)
1209 folder = d_re.group(1).split('-')
1211 resolution = folder[1]
1214 density = screen_resolutions[resolution]
1215 apk['icons_src'][density] = d_re.group(0)
1217 if apk['icons_src'].get('-1') is None:
1218 apk['icons_src']['-1'] = apk['icons_src']['160']
1220 arch_re = re.compile("^lib/(.*)/.*$")
1221 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1223 apk['nativecode'] = []
1224 apk['nativecode'].extend(sorted(list(arch)))
1226 xml = apkobject.get_android_manifest_xml()
1228 for item in xml.getElementsByTagName('uses-permission'):
1229 name = str(item.getAttribute("android:name"))
1230 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1231 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1232 permission = UsesPermission(
1236 apk['uses-permission'].append(permission)
1238 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1239 name = str(item.getAttribute("android:name"))
1240 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1241 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1242 permission_sdk_23 = UsesPermissionSdk23(
1246 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1248 for item in xml.getElementsByTagName('uses-feature'):
1249 feature = str(item.getAttribute("android:name"))
1250 if feature != "android.hardware.screen.portrait" \
1251 and feature != "android.hardware.screen.landscape":
1252 if feature.startswith("android.feature."):
1253 feature = feature[16:]
1254 apk['features'].append(feature)
1257 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1258 allow_disabled_algorithms=False, archive_bad_sig=False):
1259 """Processes the apk with the given filename in the given repo directory.
1261 This also extracts the icons.
1263 :param apkcache: current apk cache information
1264 :param apkfilename: the filename of the apk to scan
1265 :param repodir: repo directory to scan
1266 :param knownapks: known apks info
1267 :param use_date_from_apk: use date from APK (instead of current date)
1268 for newly added APKs
1269 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1270 disabled algorithms in the signature (e.g. MD5)
1271 :param archive_bad_sig: move APKs with a bad signature to the archive
1272 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1273 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1277 apkfile = os.path.join(repodir, apkfilename)
1279 cachechanged = False
1281 if apkfilename in apkcache:
1282 apk = apkcache[apkfilename]
1283 if apk.get('hash') == sha256sum(apkfile):
1284 logging.debug(_("Reading {apkfilename} from cache")
1285 .format(apkfilename=apkfilename))
1288 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1289 .format(apkfilename=apkfilename))
1292 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1295 apk = scan_apk(apkfile)
1296 except BuildException:
1297 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1298 .format(apkfilename=apkfilename))
1299 return True, None, False
1301 # Check for debuggable apks...
1302 if common.isApkAndDebuggable(apkfile):
1303 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1305 if options.rename_apks:
1306 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1307 std_short_name = os.path.join(repodir, n)
1308 if apkfile != std_short_name:
1309 if os.path.exists(std_short_name):
1310 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1311 if apkfile != std_long_name:
1312 if os.path.exists(std_long_name):
1313 dupdir = os.path.join('duplicates', repodir)
1314 if not os.path.isdir(dupdir):
1315 os.makedirs(dupdir, exist_ok=True)
1316 dupfile = os.path.join('duplicates', std_long_name)
1317 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1318 os.rename(apkfile, dupfile)
1319 return True, None, False
1321 os.rename(apkfile, std_long_name)
1322 apkfile = std_long_name
1324 os.rename(apkfile, std_short_name)
1325 apkfile = std_short_name
1326 apkfilename = apkfile[len(repodir) + 1:]
1328 apk['apkName'] = apkfilename
1329 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1330 if os.path.exists(os.path.join(repodir, srcfilename)):
1331 apk['srcname'] = srcfilename
1333 # verify the jar signature is correct, allow deprecated
1334 # algorithms only if the APK is in the archive.
1336 if not common.verify_apk_signature(apkfile):
1337 if repodir == 'archive' or allow_disabled_algorithms:
1338 if common.verify_old_apk_signature(apkfile):
1339 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1347 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1348 .format(apkfilename=apkfilename))
1349 move_apk_between_sections(repodir, 'archive', apk)
1351 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1352 .format(apkfilename=apkfilename))
1353 return True, None, False
1355 apkzip = zipfile.ZipFile(apkfile, 'r')
1357 manifest = apkzip.getinfo('AndroidManifest.xml')
1358 # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1359 if (1980, 0, 0) != manifest.date_time[0:3]:
1361 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1362 except ValueError as e:
1363 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1364 .format(apkfilename=apkfile) + str(e))
1366 # extract icons from APK zip file
1367 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1369 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1371 apkzip.close() # ensure that APK zip file gets closed
1373 # resize existing icons for densities missing in the APK
1374 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1376 if use_date_from_apk and manifest.date_time[1] != 0:
1377 default_date_param = datetime(*manifest.date_time)
1379 default_date_param = None
1381 # Record in known apks, getting the added date at the same time..
1382 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1383 default_date=default_date_param)
1385 apk['added'] = added
1387 apkcache[apkfilename] = apk
1390 return False, apk, cachechanged
1393 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1394 """Processes the apks in the given repo directory.
1396 This also extracts the icons.
1398 :param apkcache: current apk cache information
1399 :param repodir: repo directory to scan
1400 :param knownapks: known apks info
1401 :param use_date_from_apk: use date from APK (instead of current date)
1402 for newly added APKs
1403 :returns: (apks, cachechanged) where apks is a list of apk information,
1404 and cachechanged is True if the apkcache got changed.
1407 cachechanged = False
1409 for icon_dir in get_all_icon_dirs(repodir):
1410 if os.path.exists(icon_dir):
1412 shutil.rmtree(icon_dir)
1413 os.makedirs(icon_dir)
1415 os.makedirs(icon_dir)
1418 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1419 apkfilename = apkfile[len(repodir) + 1:]
1420 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1421 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1422 use_date_from_apk, ada, True)
1426 cachechanged = cachechanged or cachethis
1428 return apks, cachechanged
1431 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1433 Extracts icons from the given APK zip in various densities,
1434 saves them into given repo directory
1435 and stores their names in the APK metadata dictionary.
1437 :param icon_filename: A string representing the icon's file name
1438 :param apk: A populated dictionary containing APK metadata.
1439 Needs to have 'icons_src' key
1440 :param apkzip: An opened zipfile.ZipFile of the APK file
1441 :param repo_dir: The directory of the APK's repository
1442 :return: A list of icon densities that are missing
1444 empty_densities = []
1445 for density in screen_densities:
1446 if density not in apk['icons_src']:
1447 empty_densities.append(density)
1449 icon_src = apk['icons_src'][density]
1450 icon_dir = get_icon_dir(repo_dir, density)
1451 icon_dest = os.path.join(icon_dir, icon_filename)
1453 # Extract the icon files per density
1454 if icon_src.endswith('.xml'):
1455 png = os.path.basename(icon_src)[:-4] + '.png'
1456 for f in apkzip.namelist():
1458 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1459 if m and screen_resolutions[m.group(2)] == density:
1461 if icon_src.endswith('.xml'):
1462 empty_densities.append(density)
1465 with open(icon_dest, 'wb') as f:
1466 f.write(get_icon_bytes(apkzip, icon_src))
1467 apk['icons'][density] = icon_filename
1468 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1469 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1470 del apk['icons_src'][density]
1471 empty_densities.append(density)
1473 if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1474 icon_src = apk['icons_src']['-1']
1475 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1476 with open(icon_path, 'wb') as f:
1477 f.write(get_icon_bytes(apkzip, icon_src))
1480 im = Image.open(icon_path)
1481 dpi = px_to_dpi(im.size[0])
1482 for density in screen_densities:
1483 if density in apk['icons']:
1485 if density == screen_densities[-1] or dpi >= int(density):
1486 apk['icons'][density] = icon_filename
1487 shutil.move(icon_path,
1488 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1489 empty_densities.remove(density)
1491 except Exception as e:
1492 logging.warning(_("Failed reading {path}: {error}")
1493 .format(path=icon_path, error=e))
1495 if im and hasattr(im, 'close'):
1499 apk['icon'] = icon_filename
1501 return empty_densities
1504 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1506 Resize existing icons for densities missing in the APK to ensure all densities are available
1508 :param empty_densities: A list of icon densities that are missing
1509 :param icon_filename: A string representing the icon's file name
1510 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1511 :param repo_dir: The directory of the APK's repository
1513 # First try resizing down to not lose quality
1515 for density in screen_densities:
1516 if density not in empty_densities:
1517 last_density = density
1519 if last_density is None:
1521 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1523 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1524 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1527 fp = open(last_icon_path, 'rb')
1530 size = dpi_to_px(density)
1532 im.thumbnail((size, size), Image.ANTIALIAS)
1533 im.save(icon_path, "PNG", optimize=True,
1534 pnginfo=BLANK_PNG_INFO, icc_profile=None)
1535 empty_densities.remove(density)
1536 except Exception as e:
1537 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1542 # Then just copy from the highest resolution available
1544 for density in reversed(screen_densities):
1545 if density not in empty_densities:
1546 last_density = density
1549 if last_density is None:
1553 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1554 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1556 empty_densities.remove(density)
1558 for density in screen_densities:
1559 icon_dir = get_icon_dir(repo_dir, density)
1560 icon_dest = os.path.join(icon_dir, icon_filename)
1561 resize_icon(icon_dest, density)
1563 # Copy from icons-mdpi to icons since mdpi is the baseline density
1564 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1565 if os.path.isfile(baseline):
1566 apk['icons']['0'] = icon_filename
1567 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1570 def apply_info_from_latest_apk(apps, apks):
1572 Some information from the apks needs to be applied up to the application level.
1573 When doing this, we use the info from the most recent version's apk.
1574 We deal with figuring out when the app was added and last updated at the same time.
1576 for appid, app in apps.items():
1577 bestver = UNSET_VERSION_CODE
1579 if apk['packageName'] == appid:
1580 if apk['versionCode'] > bestver:
1581 bestver = apk['versionCode']
1585 if not app.added or apk['added'] < app.added:
1586 app.added = apk['added']
1587 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1588 app.lastUpdated = apk['added']
1591 logging.debug("Don't know when " + appid + " was added")
1592 if not app.lastUpdated:
1593 logging.debug("Don't know when " + appid + " was last updated")
1595 if bestver == UNSET_VERSION_CODE:
1597 if app.Name is None:
1598 app.Name = app.AutoName or appid
1600 logging.debug("Application " + appid + " has no packages")
1602 if app.Name is None:
1603 app.Name = bestapk['name']
1604 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1605 if app.CurrentVersionCode is None:
1606 app.CurrentVersionCode = str(bestver)
1609 def make_categories_txt(repodir, categories):
1610 '''Write a category list in the repo to allow quick access'''
1612 for cat in sorted(categories):
1613 catdata += cat + '\n'
1614 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1618 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1620 def filter_apk_list_sorted(apk_list):
1622 for apk in apk_list:
1623 if apk['packageName'] == appid:
1626 # Sort the apk list by version code. First is highest/newest.
1627 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1629 for appid, app in apps.items():
1631 if app.ArchivePolicy:
1632 keepversions = int(app.ArchivePolicy[:-9])
1634 keepversions = defaultkeepversions
1636 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1637 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1639 current_app_apks = filter_apk_list_sorted(apks)
1640 if len(current_app_apks) > keepversions:
1641 # Move back the ones we don't want.
1642 for apk in current_app_apks[keepversions:]:
1643 move_apk_between_sections(repodir, archivedir, apk)
1644 archapks.append(apk)
1647 current_app_archapks = filter_apk_list_sorted(archapks)
1648 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1650 # Move forward the ones we want again, except DisableAlgorithm
1651 for apk in current_app_archapks:
1652 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1653 move_apk_between_sections(archivedir, repodir, apk)
1654 archapks.remove(apk)
1657 if kept == keepversions:
1661 def move_apk_between_sections(from_dir, to_dir, apk):
1662 """move an APK from repo to archive or vice versa"""
1664 def _move_file(from_dir, to_dir, filename, ignore_missing):
1665 from_path = os.path.join(from_dir, filename)
1666 if ignore_missing and not os.path.exists(from_path):
1668 to_path = os.path.join(to_dir, filename)
1669 if not os.path.exists(to_dir):
1671 shutil.move(from_path, to_path)
1673 if from_dir == to_dir:
1676 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1677 _move_file(from_dir, to_dir, apk['apkName'], False)
1678 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1679 for density in all_screen_densities:
1680 from_icon_dir = get_icon_dir(from_dir, density)
1681 to_icon_dir = get_icon_dir(to_dir, density)
1682 if density not in apk.get('icons', []):
1684 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1685 if 'srcname' in apk:
1686 _move_file(from_dir, to_dir, apk['srcname'], False)
1689 def add_apks_to_per_app_repos(repodir, apks):
1690 apks_per_app = dict()
1692 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1693 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1694 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1695 apks_per_app[apk['packageName']] = apk
1697 if not os.path.exists(apk['per_app_icons']):
1698 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1699 os.makedirs(apk['per_app_icons'])
1701 apkpath = os.path.join(repodir, apk['apkName'])
1702 shutil.copy(apkpath, apk['per_app_repo'])
1703 apksigpath = apkpath + '.sig'
1704 if os.path.exists(apksigpath):
1705 shutil.copy(apksigpath, apk['per_app_repo'])
1706 apkascpath = apkpath + '.asc'
1707 if os.path.exists(apkascpath):
1708 shutil.copy(apkascpath, apk['per_app_repo'])
1711 def create_metadata_from_template(apk):
1712 '''create a new metadata file using internal or external template
1714 Generate warnings for apk's with no metadata (or create skeleton
1715 metadata files, if requested on the command line). Though the
1716 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1717 since those impose things on the metadata file made from the
1718 template: field sort order, empty field value, formatting, etc.
1722 if os.path.exists('template.yml'):
1723 with open('template.yml') as f:
1725 if 'name' in apk and apk['name'] != '':
1726 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1727 r'\1 ' + apk['name'],
1729 flags=re.IGNORECASE | re.MULTILINE)
1731 logging.warning(_('{appid} does not have a name! Using package name instead.')
1732 .format(appid=apk['packageName']))
1733 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1734 r'\1 ' + apk['packageName'],
1736 flags=re.IGNORECASE | re.MULTILINE)
1737 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1741 app['Categories'] = [os.path.basename(os.getcwd())]
1742 # include some blanks as part of the template
1743 app['AuthorName'] = ''
1746 app['IssueTracker'] = ''
1747 app['SourceCode'] = ''
1748 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1749 if 'name' in apk and apk['name'] != '':
1750 app['Name'] = apk['name']
1752 logging.warning(_('{appid} does not have a name! Using package name instead.')
1753 .format(appid=apk['packageName']))
1754 app['Name'] = apk['packageName']
1755 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1756 yaml.dump(app, f, default_flow_style=False)
1757 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1766 global config, options
1768 # Parse command line...
1769 parser = ArgumentParser()
1770 common.setup_global_opts(parser)
1771 parser.add_argument("--create-key", action="store_true", default=False,
1772 help=_("Add a repo signing key to an unsigned repo"))
1773 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1774 help=_("Add skeleton metadata files for APKs that are missing them"))
1775 parser.add_argument("--delete-unknown", action="store_true", default=False,
1776 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1777 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1778 help=_("Report on build data status"))
1779 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1780 help=_("Interactively ask about things that need updating."))
1781 parser.add_argument("-I", "--icons", action="store_true", default=False,
1782 help=_("Resize all the icons exceeding the max pixel size and exit"))
1783 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1784 help=_("Specify editor to use in interactive mode. Default " +
1785 "is {path}").format(path='/etc/alternatives/editor'))
1786 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1787 help=_("Update the wiki"))
1788 parser.add_argument("--pretty", action="store_true", default=False,
1789 help=_("Produce human-readable XML/JSON for index files"))
1790 parser.add_argument("--clean", action="store_true", default=False,
1791 help=_("Clean update - don't uses caches, reprocess all APKs"))
1792 parser.add_argument("--nosign", action="store_true", default=False,
1793 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1794 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1795 help=_("Use date from APK instead of current time for newly added APKs"))
1796 parser.add_argument("--rename-apks", action="store_true", default=False,
1797 help=_("Rename APK files that do not match package.name_123.apk"))
1798 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1799 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1800 metadata.add_metadata_arguments(parser)
1801 options = parser.parse_args()
1802 metadata.warnings_action = options.W
1804 config = common.read_config(options)
1806 if not ('jarsigner' in config and 'keytool' in config):
1807 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1810 if config['archive_older'] != 0:
1811 repodirs.append('archive')
1812 if not os.path.exists('archive'):
1816 resize_all_icons(repodirs)
1819 if options.rename_apks:
1820 options.clean = True
1822 # check that icons exist now, rather than fail at the end of `fdroid update`
1823 for k in ['repo_icon', 'archive_icon']:
1825 if not os.path.exists(config[k]):
1826 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1827 .format(name=k, path=config[k]))
1830 # if the user asks to create a keystore, do it now, reusing whatever it can
1831 if options.create_key:
1832 if os.path.exists(config['keystore']):
1833 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1834 logging.critical("\t'" + config['keystore'] + "'")
1837 if 'repo_keyalias' not in config:
1838 config['repo_keyalias'] = socket.getfqdn()
1839 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1840 if 'keydname' not in config:
1841 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1842 common.write_to_config(config, 'keydname', config['keydname'])
1843 if 'keystore' not in config:
1844 config['keystore'] = common.default_config['keystore']
1845 common.write_to_config(config, 'keystore', config['keystore'])
1847 password = common.genpassword()
1848 if 'keystorepass' not in config:
1849 config['keystorepass'] = password
1850 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1851 if 'keypass' not in config:
1852 config['keypass'] = password
1853 common.write_to_config(config, 'keypass', config['keypass'])
1854 common.genkeystore(config)
1857 apps = metadata.read_metadata()
1859 # Generate a list of categories...
1861 for app in apps.values():
1862 categories.update(app.Categories)
1864 # Read known apks data (will be updated and written back when we've finished)
1865 knownapks = common.KnownApks()
1868 apkcache = get_cache()
1870 # Delete builds for disabled apps
1871 delete_disabled_builds(apps, apkcache, repodirs)
1873 # Scan all apks in the main repo
1874 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1876 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1877 options.use_date_from_apk)
1878 cachechanged = cachechanged or fcachechanged
1881 if apk['packageName'] not in apps:
1882 if options.create_metadata:
1883 create_metadata_from_template(apk)
1884 apps = metadata.read_metadata()
1886 msg = _("{apkfilename} ({appid}) has no metadata!") \
1887 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1888 if options.delete_unknown:
1889 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1890 .format(apkfilename=apk['apkName']))
1891 rmf = os.path.join(repodirs[0], apk['apkName'])
1892 if not os.path.exists(rmf):
1893 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1897 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1899 copy_triple_t_store_metadata(apps)
1900 insert_obbs(repodirs[0], apps, apks)
1901 insert_localized_app_metadata(apps)
1902 translate_per_build_anti_features(apps, apks)
1904 # Scan the archive repo for apks as well
1905 if len(repodirs) > 1:
1906 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1912 # Apply information from latest apks to the application and update dates
1913 apply_info_from_latest_apk(apps, apks + archapks)
1915 # Sort the app list by name, then the web site doesn't have to by default.
1916 # (we had to wait until we'd scanned the apks to do this, because mostly the
1917 # name comes from there!)
1918 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1920 # APKs are placed into multiple repos based on the app package, providing
1921 # per-app subscription feeds for nightly builds and things like it
1922 if config['per_app_repos']:
1923 add_apks_to_per_app_repos(repodirs[0], apks)
1924 for appid, app in apps.items():
1925 repodir = os.path.join(appid, 'fdroid', 'repo')
1927 appdict[appid] = app
1928 if os.path.isdir(repodir):
1929 index.make(appdict, [appid], apks, repodir, False)
1931 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1934 if len(repodirs) > 1:
1935 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1937 # Make the index for the main repo...
1938 index.make(apps, sortedids, apks, repodirs[0], False)
1939 make_categories_txt(repodirs[0], categories)
1941 # If there's an archive repo, make the index for it. We already scanned it
1943 if len(repodirs) > 1:
1944 index.make(apps, sortedids, archapks, repodirs[1], True)
1946 git_remote = config.get('binary_transparency_remote')
1947 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1949 btlog.make_binary_transparency_log(repodirs)
1951 if config['update_stats']:
1952 # Update known apks info...
1953 knownapks.writeifchanged()
1955 # Generate latest apps data for widget
1956 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1958 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1960 appid = line.rstrip()
1961 data += appid + "\t"
1963 data += app.Name + "\t"
1964 if app.icon is not None:
1965 data += app.icon + "\t"
1966 data += app.License + "\n"
1967 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1971 write_cache(apkcache)
1973 # Update the wiki...
1975 update_wiki(apps, sortedids, apks + archapks)
1977 logging.info(_("Finished"))
1980 if __name__ == "__main__":