3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
38 from PIL import Image, PngImagePlugin
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")
57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
58 APK_PERMISSION_PAT = \
59 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
63 screen_resolutions = {
75 all_screen_densities = ['0'] + screen_densities
77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
83 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
85 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
97 if density == '0' or density == '65534':
98 return os.path.join(repodir, "icons")
100 return os.path.join(repodir, "icons-%s" % density)
103 def get_icon_dirs(repodir):
104 for density in screen_densities:
105 yield get_icon_dir(repodir, density)
108 def get_all_icon_dirs(repodir):
109 for density in all_screen_densities:
110 yield get_icon_dir(repodir, density)
113 def update_wiki(apps, sortedids, apks):
116 :param apps: fully populated list of all applications
117 :param apks: all apks, except...
119 logging.info("Updating wiki")
121 wikiredircat = 'App Redirects'
123 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
124 path=config['wiki_path'])
125 site.login(config['wiki_user'], config['wiki_password'])
127 generated_redirects = {}
129 for appid in sortedids:
130 app = metadata.App(apps[appid])
134 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
136 for af in sorted(app.AntiFeatures):
137 wikidata += '{{AntiFeature|' + af + '}}\n'
142 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
145 app.added.strftime('%Y-%m-%d') if app.added else '',
146 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
162 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
164 wikidata += app.Summary
165 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
167 wikidata += "=Description=\n"
168 wikidata += metadata.description_wiki(app.Description) + "\n"
170 wikidata += "=Maintainer Notes=\n"
171 if app.MaintainerNotes:
172 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
173 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
175 # Get a list of all packages for this application...
177 gotcurrentver = False
181 if apk['packageName'] == appid:
182 if str(apk['versionCode']) == app.CurrentVersionCode:
185 # Include ones we can't build, as a special case...
186 for build in app.builds:
188 if build.versionCode == app.CurrentVersionCode:
190 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
191 apklist.append({'versionCode': int(build.versionCode),
192 'versionName': build.versionName,
193 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
198 if apk['versionCode'] == int(build.versionCode):
203 apklist.append({'versionCode': int(build.versionCode),
204 'versionName': build.versionName,
205 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
207 if app.CurrentVersionCode == '0':
209 # Sort with most recent first...
210 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
212 wikidata += "=Versions=\n"
213 if len(apklist) == 0:
214 wikidata += "We currently have no versions of this app available."
215 elif not gotcurrentver:
216 wikidata += "We don't have the current version of this app."
218 wikidata += "We have the current version of this app."
219 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
220 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
221 if len(app.NoSourceSince) > 0:
222 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
223 if len(app.CurrentVersion) > 0:
224 wikidata += "The current (recommended) version is " + app.CurrentVersion
225 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
228 wikidata += "==" + apk['versionName'] + "==\n"
230 if 'buildproblem' in apk:
231 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
234 wikidata += "This version is built and signed by "
236 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
238 wikidata += "the original developer.\n\n"
239 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
241 wikidata += '\n[[Category:' + wikicat + ']]\n'
242 if len(app.NoSourceSince) > 0:
243 wikidata += '\n[[Category:Apps missing source code]]\n'
244 if validapks == 0 and not app.Disabled:
245 wikidata += '\n[[Category:Apps with no packages]]\n'
246 if cantupdate and not app.Disabled:
247 wikidata += "\n[[Category:Apps we cannot update]]\n"
248 if buildfails and not app.Disabled:
249 wikidata += "\n[[Category:Apps with failing builds]]\n"
250 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
251 wikidata += '\n[[Category:Apps to Update]]\n'
253 wikidata += '\n[[Category:Apps that are disabled]]\n'
254 if app.UpdateCheckMode == 'None' and not app.Disabled:
255 wikidata += '\n[[Category:Apps with no update check]]\n'
256 for appcat in app.Categories:
257 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
259 # We can't have underscores in the page name, even if they're in
260 # the package ID, because MediaWiki messes with them...
261 pagename = appid.replace('_', ' ')
263 # Drop a trailing newline, because mediawiki is going to drop it anyway
264 # and it we don't we'll think the page has changed when it hasn't...
265 if wikidata.endswith('\n'):
266 wikidata = wikidata[:-1]
268 generated_pages[pagename] = wikidata
270 # Make a redirect from the name to the ID too, unless there's
271 # already an existing page with the name and it isn't a redirect.
273 apppagename = app.Name.replace('_', ' ')
274 apppagename = apppagename.replace('{', '')
275 apppagename = apppagename.replace('}', ' ')
276 apppagename = apppagename.replace(':', ' ')
277 apppagename = apppagename.replace('[', ' ')
278 apppagename = apppagename.replace(']', ' ')
279 # Drop double spaces caused mostly by replacing ':' above
280 apppagename = apppagename.replace(' ', ' ')
281 for expagename in site.allpages(prefix=apppagename,
282 filterredir='nonredirects',
284 if expagename == apppagename:
286 # Another reason not to make the redirect page is if the app name
287 # is the same as it's ID, because that will overwrite the real page
288 # with an redirect to itself! (Although it seems like an odd
289 # scenario this happens a lot, e.g. where there is metadata but no
290 # builds or binaries to extract a name from.
291 if apppagename == pagename:
294 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
296 for tcat, genp in [(wikicat, generated_pages),
297 (wikiredircat, generated_redirects)]:
298 catpages = site.Pages['Category:' + tcat]
300 for page in catpages:
301 existingpages.append(page.name)
302 if page.name in genp:
303 pagetxt = page.edit()
304 if pagetxt != genp[page.name]:
305 logging.debug("Updating modified page " + page.name)
306 page.save(genp[page.name], summary='Auto-updated')
308 logging.debug("Page " + page.name + " is unchanged")
310 logging.warn("Deleting page " + page.name)
311 page.delete('No longer published')
312 for pagename, text in genp.items():
313 logging.debug("Checking " + pagename)
314 if pagename not in existingpages:
315 logging.debug("Creating page " + pagename)
317 newpage = site.Pages[pagename]
318 newpage.save(text, summary='Auto-created')
319 except Exception as e:
320 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
322 # Purge server cache to ensure counts are up to date
323 site.Pages['Repository Maintenance'].purge()
325 # Write a page with the last build log for this version code
326 wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
327 newpage = site.Pages[wiki_page_path]
329 txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
330 txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
331 txt += "* completed at " + common.get_wiki_timestamp() + '\n'
332 txt += common.get_git_describe_link()
334 txt += common.get_android_tools_version_log()
335 newpage.save(txt, summary='Run log')
336 newpage = site.Pages['update']
337 newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
340 def delete_disabled_builds(apps, apkcache, repodirs):
341 """Delete disabled build outputs.
343 :param apps: list of all applications, as per metadata.read_metadata
344 :param apkcache: current apk cache information
345 :param repodirs: the repo directories to process
347 for appid, app in apps.items():
348 for build in app['builds']:
349 if not build.disable:
351 apkfilename = common.get_release_filename(app, build)
352 iconfilename = "%s.%s.png" % (
355 for repodir in repodirs:
357 os.path.join(repodir, apkfilename),
358 os.path.join(repodir, apkfilename + '.asc'),
359 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
361 for density in all_screen_densities:
362 repo_dir = get_icon_dir(repodir, density)
363 files.append(os.path.join(repo_dir, iconfilename))
366 if os.path.exists(f):
367 logging.info("Deleting disabled build output " + f)
369 if apkfilename in apkcache:
370 del apkcache[apkfilename]
373 def resize_icon(iconpath, density):
375 if not os.path.isfile(iconpath):
380 fp = open(iconpath, 'rb')
382 size = dpi_to_px(density)
384 if any(length > size for length in im.size):
386 im.thumbnail((size, size), Image.ANTIALIAS)
387 logging.debug("%s was too large at %s - new size is %s" % (
388 iconpath, oldsize, im.size))
389 im.save(iconpath, "PNG", optimize=True,
390 pnginfo=BLANK_PNG_INFO, icc_profile=None)
392 except Exception as e:
393 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
400 def resize_all_icons(repodirs):
401 """Resize all icons that exceed the max size
403 :param repodirs: the repo directories to process
405 for repodir in repodirs:
406 for density in screen_densities:
407 icon_dir = get_icon_dir(repodir, density)
408 icon_glob = os.path.join(icon_dir, '*.png')
409 for iconpath in glob.glob(icon_glob):
410 resize_icon(iconpath, density)
414 """ Get the signing certificate of an apk. To get the same md5 has that
415 Android gets, we encode the .RSA certificate in a specific format and pass
416 it hex-encoded to the md5 digest algorithm.
418 :param apkpath: path to the apk
419 :returns: A string containing the md5 of the signature of the apk or None
420 if an error occurred.
423 with zipfile.ZipFile(apkpath, 'r') as apk:
424 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
427 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
430 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
433 cert = apk.read(certs[0])
435 cert_encoded = common.get_certificate(cert)
437 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
440 def get_cache_file():
441 return os.path.join('tmp', 'apkcache')
445 """Get the cached dict of the APK index
447 Gather information about all the apk files in the repo directory,
448 using cached data if possible. Some of the index operations take a
449 long time, like calculating the SHA-256 and verifying the APK
452 The cache is invalidated if the metadata version is different, or
453 the 'allow_disabled_algorithms' config/option is different. In
454 those cases, there is no easy way to know what has changed from
455 the cache, so just rerun the whole thing.
460 apkcachefile = get_cache_file()
461 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
462 if not options.clean and os.path.exists(apkcachefile):
463 with open(apkcachefile, 'rb') as cf:
464 apkcache = pickle.load(cf, encoding='utf-8')
465 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
466 or apkcache.get('allow_disabled_algorithms') != ada:
471 apkcache["METADATA_VERSION"] = METADATA_VERSION
472 apkcache['allow_disabled_algorithms'] = ada
477 def write_cache(apkcache):
478 apkcachefile = get_cache_file()
479 cache_path = os.path.dirname(apkcachefile)
480 if not os.path.exists(cache_path):
481 os.makedirs(cache_path)
482 with open(apkcachefile, 'wb') as cf:
483 pickle.dump(apkcache, cf)
486 def get_icon_bytes(apkzip, iconsrc):
487 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
489 return apkzip.read(iconsrc)
491 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
494 def sha256sum(filename):
495 '''Calculate the sha256 of the given file'''
496 sha = hashlib.sha256()
497 with open(filename, 'rb') as f:
503 return sha.hexdigest()
506 def has_known_vulnerability(filename):
507 """checks for known vulnerabilities in the APK
509 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
510 version. Google also enforces this:
511 https://support.google.com/faqs/answer/6376725?hl=en
513 Checks whether there are more than one classes.dex or AndroidManifest.xml
514 files, which is invalid and an essential part of the "Master Key" attack.
515 http://www.saurik.com/id/17
517 Janus is similar to Master Key but is perhaps easier to scan for.
518 https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
523 # statically load this pattern
524 if not hasattr(has_known_vulnerability, "pattern"):
525 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
527 with open(filename.encode(), 'rb') as fp:
529 if first4 != b'\x50\x4b\x03\x04':
530 raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
531 .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
532 + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
535 with zipfile.ZipFile(filename) as zf:
536 for name in zf.namelist():
537 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
540 chunk = lib.read(4096)
543 m = has_known_vulnerability.pattern.search(chunk)
545 version = m.group(1).decode('ascii')
546 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
547 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
548 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
549 logging.debug(_('"{path}" contains recent {name} ({version})')
550 .format(path=filename, name=name, version=version))
552 logging.warning(_('"{path}" contains outdated {name} ({version})')
553 .format(path=filename, name=name, version=version))
556 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
557 if name in files_in_apk:
558 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
559 .format(apkfilename=filename, name=name))
561 files_in_apk.add(name)
565 def insert_obbs(repodir, apps, apks):
566 """Scans the .obb files in a given repo directory and adds them to the
567 relevant APK instances. OBB files have versionCodes like APK
568 files, and they are loosely associated. If there is an OBB file
569 present, then any APK with the same or higher versionCode will use
570 that OBB file. There are two OBB types: main and patch, each APK
571 can only have only have one of each.
573 https://developer.android.com/google/play/expansion-files.html
575 :param repodir: repo directory to scan
576 :param apps: list of current, valid apps
577 :param apks: current information on all APKs
581 def obbWarnDelete(f, msg):
582 logging.warning(msg + ' ' + f)
583 if options.delete_unknown:
584 logging.error(_("Deleting unknown file: {path}").format(path=f))
588 java_Integer_MIN_VALUE = -pow(2, 31)
589 currentPackageNames = apps.keys()
590 for f in glob.glob(os.path.join(repodir, '*.obb')):
591 obbfile = os.path.basename(f)
592 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
593 chunks = obbfile.split('.')
594 if chunks[0] != 'main' and chunks[0] != 'patch':
595 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
597 if not re.match(r'^-?[0-9]+$', chunks[1]):
598 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
599 .format(name=chunks[0]))
601 versionCode = int(chunks[1])
602 packagename = ".".join(chunks[2:-1])
604 highestVersionCode = java_Integer_MIN_VALUE
605 if packagename not in currentPackageNames:
606 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
609 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
610 highestVersionCode = apk['versionCode']
611 if versionCode > highestVersionCode:
612 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
613 .format(integer=str(versionCode)))
615 obbsha256 = sha256sum(f)
616 obbs.append((packagename, versionCode, obbfile, obbsha256))
619 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
620 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
621 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
622 apk['obbMainFile'] = obbfile
623 apk['obbMainFileSha256'] = obbsha256
624 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
625 apk['obbPatchFile'] = obbfile
626 apk['obbPatchFileSha256'] = obbsha256
627 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
631 def translate_per_build_anti_features(apps, apks):
632 """Grab the anti-features list from the build metadata
634 For most Anti-Features, they are really most applicable per-APK,
635 not for an app. An app can fix a vulnerability, add/remove
636 tracking, etc. This reads the 'antifeatures' list from the Build
637 entries in the fdroiddata metadata file, then transforms it into
638 the 'antiFeatures' list of unique items for the index.
640 The field key is all lower case in the metadata file to match the
641 rest of the Build fields. It is 'antiFeatures' camel case in the
642 implementation, index, and fdroidclient since it is translated
643 from the build 'antifeatures' field, not directly included.
647 antiFeatures = dict()
648 for packageName, app in apps.items():
650 for build in app['builds']:
651 afl = build.get('antifeatures')
653 d[int(build.versionCode)] = afl
655 antiFeatures[packageName] = d
658 d = antiFeatures.get(apk['packageName'])
660 afl = d.get(apk['versionCode'])
662 apk['antiFeatures'].update(afl)
665 def _get_localized_dict(app, locale):
666 '''get the dict to add localized store metadata to'''
667 if 'localized' not in app:
668 app['localized'] = collections.OrderedDict()
669 if locale not in app['localized']:
670 app['localized'][locale] = collections.OrderedDict()
671 return app['localized'][locale]
674 def _set_localized_text_entry(app, locale, key, f):
675 limit = config['char_limits'][key]
676 localized = _get_localized_dict(app, locale)
678 text = fp.read()[:limit]
680 localized[key] = text
683 def _set_author_entry(app, key, f):
684 limit = config['char_limits']['author']
686 text = fp.read()[:limit]
691 def _strip_and_copy_image(inpath, outpath):
692 """Remove any metadata from image and copy it to new path
694 Sadly, image metadata like EXIF can be used to exploit devices.
695 It is not used at all in the F-Droid ecosystem, so its much safer
696 just to remove it entirely.
700 extension = common.get_extension(inpath)[1]
701 if os.path.isdir(outpath):
702 outpath = os.path.join(outpath, os.path.basename(inpath))
703 if extension == 'png':
704 with open(inpath, 'rb') as fp:
705 in_image = Image.open(fp)
706 in_image.save(outpath, "PNG", optimize=True,
707 pnginfo=BLANK_PNG_INFO, icc_profile=None)
708 elif extension == 'jpg' or extension == 'jpeg':
709 with open(inpath, 'rb') as fp:
710 in_image = Image.open(fp)
711 data = list(in_image.getdata())
712 out_image = Image.new(in_image.mode, in_image.size)
713 out_image.putdata(data)
714 out_image.save(outpath, "JPEG", optimize=True)
716 raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
717 .format(extension=extension))
720 def copy_triple_t_store_metadata(apps):
721 """Include store metadata from the app's source repo
723 The Triple-T Gradle Play Publisher is a plugin that has a standard
724 file layout for all of the metadata and graphics that the Google
725 Play Store accepts. Since F-Droid has the git repo, it can just
726 pluck those files directly. This method reads any text files into
727 the app dict, then copies any graphics into the fdroid repo
730 This needs to be run before insert_localized_app_metadata() so that
731 the graphics files that are copied into the fdroid repo get
734 https://github.com/Triple-T/gradle-play-publisher#upload-images
735 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
739 if not os.path.isdir('build'):
740 return # nothing to do
742 for packageName, app in apps.items():
743 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
744 logging.debug('Triple-T Gradle Play Publisher: ' + d)
745 for root, dirs, files in os.walk(d):
746 segments = root.split('/')
747 locale = segments[-2]
749 if f == 'fulldescription':
750 _set_localized_text_entry(app, locale, 'description',
751 os.path.join(root, f))
753 elif f == 'shortdescription':
754 _set_localized_text_entry(app, locale, 'summary',
755 os.path.join(root, f))
758 _set_localized_text_entry(app, locale, 'name',
759 os.path.join(root, f))
762 _set_localized_text_entry(app, locale, 'video',
763 os.path.join(root, f))
765 elif f == 'whatsnew':
766 _set_localized_text_entry(app, segments[-1], 'whatsNew',
767 os.path.join(root, f))
769 elif f == 'contactEmail':
770 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
772 elif f == 'contactPhone':
773 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
775 elif f == 'contactWebsite':
776 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
779 base, extension = common.get_extension(f)
780 dirname = os.path.basename(root)
781 if extension in ALLOWED_EXTENSIONS \
782 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
783 if segments[-2] == 'listing':
784 locale = segments[-3]
786 locale = segments[-2]
787 destdir = os.path.join('repo', packageName, locale, dirname)
788 os.makedirs(destdir, mode=0o755, exist_ok=True)
789 sourcefile = os.path.join(root, f)
790 destfile = os.path.join(destdir, os.path.basename(f))
791 logging.debug('copying ' + sourcefile + ' ' + destfile)
792 _strip_and_copy_image(sourcefile, destfile)
795 def insert_localized_app_metadata(apps):
796 """scans standard locations for graphics and localized text
798 Scans for localized description files, store graphics, and
799 screenshot PNG files in statically defined screenshots directory
800 and adds them to the app metadata. The screenshots and graphic
801 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
802 and must be in the following layout:
803 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
805 repo/packageName/locale/featureGraphic.png
806 repo/packageName/locale/phoneScreenshots/1.png
807 repo/packageName/locale/phoneScreenshots/2.png
809 The changelog files must be text files named with the versionCode
810 ending with ".txt" and must be in the following layout:
811 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
813 repo/packageName/locale/changelogs/12345.txt
815 This will scan the each app's source repo then the metadata/ dir
816 for these standard locations of changelog files. If it finds
817 them, they will be added to the dict of all packages, with the
818 versions in the metadata/ folder taking precendence over the what
819 is in the app's source repo.
821 Where "packageName" is the app's packageName and "locale" is the locale
822 of the graphics, e.g. what language they are in, using the IETF RFC5646
823 format (en-US, fr-CA, es-MX, etc).
825 This will also scan the app's git for a fastlane folder, and the
826 metadata/ folder and the apps' source repos for standard locations
827 of graphic and screenshot files. If it finds them, it will copy
828 them into the repo. The fastlane files follow this pattern:
829 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
833 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
834 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
835 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
836 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
838 for srcd in sorted(sourcedirs):
839 if not os.path.isdir(srcd):
841 for root, dirs, files in os.walk(srcd):
842 segments = root.split('/')
843 packageName = segments[1]
844 if packageName not in apps:
845 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
847 locale = segments[-1]
848 destdir = os.path.join('repo', packageName, locale)
850 # flavours specified in build receipt
852 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
853 and 'gradle' in apps[packageName].builds[-1]:
854 build_flavours = apps[packageName].builds[-1].gradle
856 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
857 logging.debug("ignoring due to wrong flavour")
861 if f in ('description.txt', 'full_description.txt'):
862 _set_localized_text_entry(apps[packageName], locale, 'description',
863 os.path.join(root, f))
865 elif f in ('summary.txt', 'short_description.txt'):
866 _set_localized_text_entry(apps[packageName], locale, 'summary',
867 os.path.join(root, f))
869 elif f in ('name.txt', 'title.txt'):
870 _set_localized_text_entry(apps[packageName], locale, 'name',
871 os.path.join(root, f))
873 elif f == 'video.txt':
874 _set_localized_text_entry(apps[packageName], locale, 'video',
875 os.path.join(root, f))
877 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
878 locale = segments[-2]
879 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
880 os.path.join(root, f))
883 base, extension = common.get_extension(f)
884 if locale == 'images':
885 locale = segments[-2]
886 destdir = os.path.join('repo', packageName, locale)
887 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
888 os.makedirs(destdir, mode=0o755, exist_ok=True)
889 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
890 _strip_and_copy_image(os.path.join(root, f), destdir)
892 if d in SCREENSHOT_DIRS:
893 if locale == 'images':
894 locale = segments[-2]
895 destdir = os.path.join('repo', packageName, locale)
896 for f in glob.glob(os.path.join(root, d, '*.*')):
897 _ignored, extension = common.get_extension(f)
898 if extension in ALLOWED_EXTENSIONS:
899 screenshotdestdir = os.path.join(destdir, d)
900 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
901 logging.debug('copying ' + f + ' ' + screenshotdestdir)
902 _strip_and_copy_image(f, screenshotdestdir)
904 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
906 if not os.path.isdir(d):
908 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
909 if not os.path.isfile(f):
911 segments = f.split('/')
912 packageName = segments[1]
914 screenshotdir = segments[3]
915 filename = os.path.basename(f)
916 base, extension = common.get_extension(filename)
918 if packageName not in apps:
919 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
920 .format(path=filename, name=packageName))
922 graphics = _get_localized_dict(apps[packageName], locale)
924 if extension not in ALLOWED_EXTENSIONS:
925 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
926 elif base in GRAPHIC_NAMES:
927 # there can only be zero or one of these per locale
928 graphics[base] = filename
929 elif screenshotdir in SCREENSHOT_DIRS:
930 # there can any number of these per locale
931 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
932 if screenshotdir not in graphics:
933 graphics[screenshotdir] = []
934 graphics[screenshotdir].append(filename)
936 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
940 """Scan a repo for all files with an extension except APK/OBB
942 :param apkcache: current cached info about all repo files
943 :param repodir: repo directory to scan
944 :param knownapks: list of all known files, as per metadata.read_metadata
945 :param use_date_from_file: use date from file (instead of current date)
946 for newly added files
951 repodir = repodir.encode('utf-8')
952 for name in os.listdir(repodir):
953 file_extension = common.get_file_extension(name)
954 if file_extension == 'apk' or file_extension == 'obb':
956 filename = os.path.join(repodir, name)
957 name_utf8 = name.decode('utf-8')
958 if filename.endswith(b'_src.tar.gz'):
959 logging.debug(_('skipping source tarball: {path}')
960 .format(path=filename.decode('utf-8')))
962 if not common.is_repo_file(filename):
964 stat = os.stat(filename)
965 if stat.st_size == 0:
966 raise FDroidException(_('{path} is zero size!')
967 .format(path=filename))
969 shasum = sha256sum(filename)
972 repo_file = apkcache[name]
973 # added time is cached as tuple but used here as datetime instance
974 if 'added' in repo_file:
975 a = repo_file['added']
976 if isinstance(a, datetime):
977 repo_file['added'] = a
979 repo_file['added'] = datetime(*a[:6])
980 if repo_file.get('hash') == shasum:
981 logging.debug(_("Reading {apkfilename} from cache")
982 .format(apkfilename=name_utf8))
985 logging.debug(_("Ignoring stale cache data for {apkfilename}")
986 .format(apkfilename=name_utf8))
989 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
990 repo_file = collections.OrderedDict()
991 repo_file['name'] = os.path.splitext(name_utf8)[0]
992 # TODO rename apkname globally to something more generic
993 repo_file['apkName'] = name_utf8
994 repo_file['hash'] = shasum
995 repo_file['hashType'] = 'sha256'
996 repo_file['versionCode'] = 0
997 repo_file['versionName'] = shasum
998 # the static ID is the SHA256 unless it is set in the metadata
999 repo_file['packageName'] = shasum
1001 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1003 repo_file['packageName'] = m.group(1)
1004 repo_file['versionCode'] = int(m.group(2))
1005 srcfilename = name + b'_src.tar.gz'
1006 if os.path.exists(os.path.join(repodir, srcfilename)):
1007 repo_file['srcname'] = srcfilename.decode('utf-8')
1008 repo_file['size'] = stat.st_size
1010 apkcache[name] = repo_file
1013 if use_date_from_file:
1014 timestamp = stat.st_ctime
1015 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1017 default_date_param = None
1019 # Record in knownapks, getting the added date at the same time..
1020 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021 default_date=default_date_param)
1023 repo_file['added'] = added
1025 repo_files.append(repo_file)
1027 return repo_files, cachechanged
1030 def scan_apk(apk_file):
1032 Scans an APK file and returns dictionary with metadata of the APK.
1034 Attention: This does *not* verify that the APK signature is correct.
1036 :param apk_file: The (ideally absolute) path to the APK file
1037 :raises BuildException
1038 :return A dict containing APK metadata
1041 'hash': sha256sum(apk_file),
1042 'hashType': 'sha256',
1043 'uses-permission': [],
1044 'uses-permission-sdk-23': [],
1048 'antiFeatures': set(),
1051 if common.use_androguard():
1052 scan_apk_androguard(apk, apk_file)
1054 scan_apk_aapt(apk, apk_file)
1056 # Get the signature, or rather the signing key fingerprints
1057 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058 apk['sig'] = getsig(apk_file)
1060 raise BuildException("Failed to get apk signature")
1061 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1063 if not apk.get('signer'):
1064 raise BuildException("Failed to get apk signing key fingerprint")
1066 # Get size of the APK
1067 apk['size'] = os.path.getsize(apk_file)
1069 if 'minSdkVersion' not in apk:
1070 logging.warning("No SDK version information found in {0}".format(apk_file))
1071 apk['minSdkVersion'] = 3 # aapt defaults to 3 as the min
1072 if 'targetSdkVersion' not in apk:
1073 apk['targetSdkVersion'] = apk['minSdkVersion']
1075 # Check for known vulnerabilities
1076 if has_known_vulnerability(apk_file):
1077 apk['antiFeatures'].add('KnownVuln')
1082 def _get_apk_icons_src(apkfile, icon_name):
1083 """Extract the paths to the app icon in all available densities
1087 density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$')
1088 with zipfile.ZipFile(apkfile) as zf:
1089 for filename in zf.namelist():
1090 m = density_re.match(filename)
1092 folder = m.group(1).split('-')
1094 density = screen_resolutions[folder[1]]
1097 icons_src[density] = m.group(0)
1098 if icons_src.get('-1') is None:
1099 icons_src['-1'] = icons_src['160']
1103 def scan_apk_aapt(apk, apkfile):
1104 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1105 if p.returncode != 0:
1106 if options.delete_unknown:
1107 if os.path.exists(apkfile):
1108 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1111 logging.error("Could not find {0} to remove it".format(apkfile))
1113 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1114 raise BuildException(_("Invalid APK"))
1116 for line in p.output.splitlines():
1117 if line.startswith("package:"):
1119 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1120 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1121 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1122 except Exception as e:
1123 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1124 elif line.startswith("application:"):
1125 m = re.match(APK_LABEL_ICON_PAT, line)
1127 apk['name'] = m.group(1)
1128 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1129 elif not apk.get('name') and line.startswith("launchable-activity:"):
1130 # Only use launchable-activity as fallback to application
1131 apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1132 elif line.startswith("sdkVersion:"):
1133 m = re.match(APK_SDK_VERSION_PAT, line)
1135 logging.error(line.replace('sdkVersion:', '')
1136 + ' is not a valid minSdkVersion!')
1138 apk['minSdkVersion'] = m.group(1)
1139 elif line.startswith("targetSdkVersion:"):
1140 m = re.match(APK_SDK_VERSION_PAT, line)
1142 logging.error(line.replace('targetSdkVersion:', '')
1143 + ' is not a valid targetSdkVersion!')
1145 apk['targetSdkVersion'] = m.group(1)
1146 elif line.startswith("maxSdkVersion:"):
1147 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1148 elif line.startswith("native-code:"):
1149 apk['nativecode'] = []
1150 for arch in line[13:].split(' '):
1151 apk['nativecode'].append(arch[1:-1])
1152 elif line.startswith('uses-permission:'):
1153 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1154 if perm_match['maxSdkVersion']:
1155 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1156 permission = UsesPermission(
1158 perm_match['maxSdkVersion']
1161 apk['uses-permission'].append(permission)
1162 elif line.startswith('uses-permission-sdk-23:'):
1163 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1164 if perm_match['maxSdkVersion']:
1165 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1166 permission_sdk_23 = UsesPermissionSdk23(
1168 perm_match['maxSdkVersion']
1171 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1173 elif line.startswith('uses-feature:'):
1174 feature = re.match(APK_FEATURE_PAT, line).group(1)
1175 # Filter out this, it's only added with the latest SDK tools and
1176 # causes problems for lots of apps.
1177 if feature != "android.hardware.screen.portrait" \
1178 and feature != "android.hardware.screen.landscape":
1179 if feature.startswith("android.feature."):
1180 feature = feature[16:]
1181 apk['features'].add(feature)
1182 apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1185 def scan_apk_androguard(apk, apkfile):
1187 from androguard.core.bytecodes.apk import APK
1188 apkobject = APK(apkfile)
1189 if apkobject.is_valid_APK():
1190 arsc = apkobject.get_android_resources()
1192 if options.delete_unknown:
1193 if os.path.exists(apkfile):
1194 logging.error(_("Failed to get apk information, deleting {path}")
1195 .format(path=apkfile))
1198 logging.error(_("Could not find {path} to remove it")
1199 .format(path=apkfile))
1201 logging.error(_("Failed to get apk information, skipping {path}")
1202 .format(path=apkfile))
1203 raise BuildException(_("Invalid APK"))
1205 raise FDroidException("androguard library is not installed and aapt not present")
1206 except FileNotFoundError:
1207 logging.error(_("Could not open apk file for analysis"))
1208 raise BuildException(_("Invalid APK"))
1210 apk['packageName'] = apkobject.get_package()
1211 apk['versionCode'] = int(apkobject.get_androidversion_code())
1212 apk['versionName'] = apkobject.get_androidversion_name()
1213 if apk['versionName'][0] == "@":
1214 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1215 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1216 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1217 apk['name'] = apkobject.get_app_name()
1219 if apkobject.get_max_sdk_version() is not None:
1220 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1221 if apkobject.get_min_sdk_version() is not None:
1222 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1223 if apkobject.get_target_sdk_version() is not None:
1224 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1226 icon_id_str = apkobject.get_element("application", "icon")
1228 icon_id = int(icon_id_str.replace("@", "0x"), 16)
1229 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1230 apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1232 arch_re = re.compile("^lib/(.*)/.*$")
1233 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1235 apk['nativecode'] = []
1236 apk['nativecode'].extend(sorted(list(arch)))
1238 xml = apkobject.get_android_manifest_xml()
1240 for item in xml.findall('uses-permission'):
1241 name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1242 maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1243 maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1244 permission = UsesPermission(
1248 apk['uses-permission'].append(permission)
1249 for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1250 permission = UsesPermission(
1254 apk['uses-permission'].append(permission)
1256 for item in xml.findall('uses-permission-sdk-23'):
1257 name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1258 maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1259 maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1260 permission_sdk_23 = UsesPermissionSdk23(
1264 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1266 for item in xml.findall('uses-feature'):
1267 feature = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1268 if feature != "android.hardware.screen.portrait" \
1269 and feature != "android.hardware.screen.landscape":
1270 if feature.startswith("android.feature."):
1271 feature = feature[16:]
1272 required = item.attrib.get('{' + xml.nsmap['android'] + '}required')
1273 if required is None or required == 'true':
1274 apk['features'].append(feature)
1277 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1278 allow_disabled_algorithms=False, archive_bad_sig=False):
1279 """Processes the apk with the given filename in the given repo directory.
1281 This also extracts the icons.
1283 :param apkcache: current apk cache information
1284 :param apkfilename: the filename of the apk to scan
1285 :param repodir: repo directory to scan
1286 :param knownapks: known apks info
1287 :param use_date_from_apk: use date from APK (instead of current date)
1288 for newly added APKs
1289 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1290 disabled algorithms in the signature (e.g. MD5)
1291 :param archive_bad_sig: move APKs with a bad signature to the archive
1292 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1293 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1297 apkfile = os.path.join(repodir, apkfilename)
1299 cachechanged = False
1301 if apkfilename in apkcache:
1302 apk = apkcache[apkfilename]
1303 if apk.get('hash') == sha256sum(apkfile):
1304 logging.debug(_("Reading {apkfilename} from cache")
1305 .format(apkfilename=apkfilename))
1308 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1309 .format(apkfilename=apkfilename))
1312 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1315 apk = scan_apk(apkfile)
1316 except BuildException:
1317 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1318 .format(apkfilename=apkfilename))
1319 return True, None, False
1321 # Check for debuggable apks...
1322 if common.is_apk_and_debuggable(apkfile):
1323 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1325 if options.rename_apks:
1326 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1327 std_short_name = os.path.join(repodir, n)
1328 if apkfile != std_short_name:
1329 if os.path.exists(std_short_name):
1330 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1331 if apkfile != std_long_name:
1332 if os.path.exists(std_long_name):
1333 dupdir = os.path.join('duplicates', repodir)
1334 if not os.path.isdir(dupdir):
1335 os.makedirs(dupdir, exist_ok=True)
1336 dupfile = os.path.join('duplicates', std_long_name)
1337 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1338 os.rename(apkfile, dupfile)
1339 return True, None, False
1341 os.rename(apkfile, std_long_name)
1342 apkfile = std_long_name
1344 os.rename(apkfile, std_short_name)
1345 apkfile = std_short_name
1346 apkfilename = apkfile[len(repodir) + 1:]
1348 apk['apkName'] = apkfilename
1349 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1350 if os.path.exists(os.path.join(repodir, srcfilename)):
1351 apk['srcname'] = srcfilename
1353 # verify the jar signature is correct, allow deprecated
1354 # algorithms only if the APK is in the archive.
1356 if not common.verify_apk_signature(apkfile):
1357 if repodir == 'archive' or allow_disabled_algorithms:
1358 if common.verify_old_apk_signature(apkfile):
1359 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1367 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1368 .format(apkfilename=apkfilename))
1369 move_apk_between_sections(repodir, 'archive', apk)
1371 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1372 .format(apkfilename=apkfilename))
1373 return True, None, False
1375 apkzip = zipfile.ZipFile(apkfile, 'r')
1377 manifest = apkzip.getinfo('AndroidManifest.xml')
1378 # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1379 if (1980, 0, 0) != manifest.date_time[0:3]:
1381 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1382 except ValueError as e:
1383 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1384 .format(apkfilename=apkfile) + str(e))
1386 # extract icons from APK zip file
1387 iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1389 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1391 apkzip.close() # ensure that APK zip file gets closed
1393 # resize existing icons for densities missing in the APK
1394 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1396 if use_date_from_apk and manifest.date_time[1] != 0:
1397 default_date_param = datetime(*manifest.date_time)
1399 default_date_param = None
1401 # Record in known apks, getting the added date at the same time..
1402 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1403 default_date=default_date_param)
1405 apk['added'] = added
1407 apkcache[apkfilename] = apk
1410 return False, apk, cachechanged
1413 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1414 """Processes the apks in the given repo directory.
1416 This also extracts the icons.
1418 :param apkcache: current apk cache information
1419 :param repodir: repo directory to scan
1420 :param knownapks: known apks info
1421 :param use_date_from_apk: use date from APK (instead of current date)
1422 for newly added APKs
1423 :returns: (apks, cachechanged) where apks is a list of apk information,
1424 and cachechanged is True if the apkcache got changed.
1427 cachechanged = False
1429 for icon_dir in get_all_icon_dirs(repodir):
1430 if os.path.exists(icon_dir):
1432 shutil.rmtree(icon_dir)
1433 os.makedirs(icon_dir)
1435 os.makedirs(icon_dir)
1438 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1439 apkfilename = apkfile[len(repodir) + 1:]
1440 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1441 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1442 use_date_from_apk, ada, True)
1446 cachechanged = cachechanged or cachethis
1448 return apks, cachechanged
1451 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1452 """Extracts PNG icons from an APK with the supported pixel densities
1454 Extracts icons from the given APK zip in various densities, saves
1455 them into given repo directory and stores their names in the APK
1456 metadata dictionary. If the icon is an XML icon, then this tries
1457 to find PNG icon that can replace it.
1459 :param icon_filename: A string representing the icon's file name
1460 :param apk: A populated dictionary containing APK metadata.
1461 Needs to have 'icons_src' key
1462 :param apkzip: An opened zipfile.ZipFile of the APK file
1463 :param repo_dir: The directory of the APK's repository
1464 :return: A list of icon densities that are missing
1467 res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1469 for f in apkzip.namelist():
1470 m = res_name_re.match(f)
1471 if m and m.group(4) == 'png':
1472 density = screen_resolutions[m.group(2)]
1473 pngs[m.group(3) + '/' + density] = m.group(0)
1476 empty_densities = []
1477 for density in screen_densities:
1478 if density not in apk['icons_src']:
1479 empty_densities.append(density)
1481 icon_src = apk['icons_src'][density]
1482 icon_dir = get_icon_dir(repo_dir, density)
1485 # Extract the icon files per density
1486 if icon_src.endswith('.xml'):
1487 m = res_name_re.match(icon_src)
1489 name = pngs.get(m.group(3) + '/' + str(density))
1492 if icon_src.endswith('.xml'):
1493 empty_densities.append(density)
1495 icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1498 with open(icon_dest, 'wb') as f:
1499 f.write(get_icon_bytes(apkzip, icon_src))
1500 apk['icons'][density] = icon_filename + icon_type
1501 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1502 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1503 del apk['icons_src'][density]
1504 empty_densities.append(density)
1506 # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1507 if '-1' in apk['icons_src']:
1508 icon_src = apk['icons_src']['-1']
1509 icon_type = icon_src[-4:]
1510 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1511 with open(icon_path, 'wb') as f:
1512 f.write(get_icon_bytes(apkzip, icon_src))
1513 if icon_type == '.png':
1516 im = Image.open(icon_path)
1517 dpi = px_to_dpi(im.size[0])
1518 for density in screen_densities:
1519 if density in apk['icons']:
1521 if density == screen_densities[-1] or dpi >= int(density):
1522 apk['icons'][density] = icon_filename + icon_type
1523 shutil.move(icon_path,
1524 os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1525 empty_densities.remove(density)
1527 except Exception as e:
1528 logging.warning(_("Failed reading {path}: {error}")
1529 .format(path=icon_path, error=e))
1531 if im and hasattr(im, 'close'):
1535 apk['icon'] = icon_filename + icon_type
1537 return empty_densities
1540 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1542 Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1544 :param empty_densities: A list of icon densities that are missing
1545 :param icon_filename: A string representing the icon's file name
1546 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1547 :param repo_dir: The directory of the APK's repository
1550 icon_filename += '.png'
1551 # First try resizing down to not lose quality
1553 for density in screen_densities:
1554 if density == '65534': # not possible to generate 'anydpi' from other densities
1556 if density not in empty_densities:
1557 last_density = density
1559 if last_density is None:
1561 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1563 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1564 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1567 fp = open(last_icon_path, 'rb')
1570 size = dpi_to_px(density)
1572 im.thumbnail((size, size), Image.ANTIALIAS)
1573 im.save(icon_path, "PNG", optimize=True,
1574 pnginfo=BLANK_PNG_INFO, icc_profile=None)
1575 empty_densities.remove(density)
1576 except Exception as e:
1577 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1582 # Then just copy from the highest resolution available
1584 for density in reversed(screen_densities):
1585 if density not in empty_densities:
1586 last_density = density
1589 if last_density is None:
1593 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1594 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1596 empty_densities.remove(density)
1598 for density in screen_densities:
1599 icon_dir = get_icon_dir(repo_dir, density)
1600 icon_dest = os.path.join(icon_dir, icon_filename)
1601 resize_icon(icon_dest, density)
1603 # Copy from icons-mdpi to icons since mdpi is the baseline density
1604 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1605 if os.path.isfile(baseline):
1606 apk['icons']['0'] = icon_filename
1607 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1610 def apply_info_from_latest_apk(apps, apks):
1612 Some information from the apks needs to be applied up to the application level.
1613 When doing this, we use the info from the most recent version's apk.
1614 We deal with figuring out when the app was added and last updated at the same time.
1616 for appid, app in apps.items():
1617 bestver = UNSET_VERSION_CODE
1619 if apk['packageName'] == appid:
1620 if apk['versionCode'] > bestver:
1621 bestver = apk['versionCode']
1625 if not app.added or apk['added'] < app.added:
1626 app.added = apk['added']
1627 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1628 app.lastUpdated = apk['added']
1631 logging.debug("Don't know when " + appid + " was added")
1632 if not app.lastUpdated:
1633 logging.debug("Don't know when " + appid + " was last updated")
1635 if bestver == UNSET_VERSION_CODE:
1637 if app.Name is None:
1638 app.Name = app.AutoName or appid
1640 logging.debug("Application " + appid + " has no packages")
1642 if app.Name is None:
1643 app.Name = bestapk['name']
1644 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1645 if app.CurrentVersionCode is None:
1646 app.CurrentVersionCode = str(bestver)
1649 def make_categories_txt(repodir, categories):
1650 '''Write a category list in the repo to allow quick access'''
1652 for cat in sorted(categories):
1653 catdata += cat + '\n'
1654 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1658 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1660 def filter_apk_list_sorted(apk_list):
1662 for apk in apk_list:
1663 if apk['packageName'] == appid:
1666 # Sort the apk list by version code. First is highest/newest.
1667 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1669 for appid, app in apps.items():
1671 if app.ArchivePolicy:
1672 keepversions = int(app.ArchivePolicy[:-9])
1674 keepversions = defaultkeepversions
1676 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1677 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1679 current_app_apks = filter_apk_list_sorted(apks)
1680 if len(current_app_apks) > keepversions:
1681 # Move back the ones we don't want.
1682 for apk in current_app_apks[keepversions:]:
1683 move_apk_between_sections(repodir, archivedir, apk)
1684 archapks.append(apk)
1687 current_app_archapks = filter_apk_list_sorted(archapks)
1688 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1690 # Move forward the ones we want again, except DisableAlgorithm
1691 for apk in current_app_archapks:
1692 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1693 move_apk_between_sections(archivedir, repodir, apk)
1694 archapks.remove(apk)
1697 if kept == keepversions:
1701 def move_apk_between_sections(from_dir, to_dir, apk):
1702 """move an APK from repo to archive or vice versa"""
1704 def _move_file(from_dir, to_dir, filename, ignore_missing):
1705 from_path = os.path.join(from_dir, filename)
1706 if ignore_missing and not os.path.exists(from_path):
1708 to_path = os.path.join(to_dir, filename)
1709 if not os.path.exists(to_dir):
1711 shutil.move(from_path, to_path)
1713 if from_dir == to_dir:
1716 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1717 _move_file(from_dir, to_dir, apk['apkName'], False)
1718 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1719 for density in all_screen_densities:
1720 from_icon_dir = get_icon_dir(from_dir, density)
1721 to_icon_dir = get_icon_dir(to_dir, density)
1722 if density not in apk.get('icons', []):
1724 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1725 if 'srcname' in apk:
1726 _move_file(from_dir, to_dir, apk['srcname'], False)
1729 def add_apks_to_per_app_repos(repodir, apks):
1730 apks_per_app = dict()
1732 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1733 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1734 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1735 apks_per_app[apk['packageName']] = apk
1737 if not os.path.exists(apk['per_app_icons']):
1738 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1739 os.makedirs(apk['per_app_icons'])
1741 apkpath = os.path.join(repodir, apk['apkName'])
1742 shutil.copy(apkpath, apk['per_app_repo'])
1743 apksigpath = apkpath + '.sig'
1744 if os.path.exists(apksigpath):
1745 shutil.copy(apksigpath, apk['per_app_repo'])
1746 apkascpath = apkpath + '.asc'
1747 if os.path.exists(apkascpath):
1748 shutil.copy(apkascpath, apk['per_app_repo'])
1751 def create_metadata_from_template(apk):
1752 '''create a new metadata file using internal or external template
1754 Generate warnings for apk's with no metadata (or create skeleton
1755 metadata files, if requested on the command line). Though the
1756 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1757 since those impose things on the metadata file made from the
1758 template: field sort order, empty field value, formatting, etc.
1762 if os.path.exists('template.yml'):
1763 with open('template.yml') as f:
1765 if 'name' in apk and apk['name'] != '':
1766 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1767 r'\1 ' + apk['name'],
1769 flags=re.IGNORECASE | re.MULTILINE)
1771 logging.warning(_('{appid} does not have a name! Using package name instead.')
1772 .format(appid=apk['packageName']))
1773 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1774 r'\1 ' + apk['packageName'],
1776 flags=re.IGNORECASE | re.MULTILINE)
1777 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1781 app['Categories'] = [os.path.basename(os.getcwd())]
1782 # include some blanks as part of the template
1783 app['AuthorName'] = ''
1786 app['IssueTracker'] = ''
1787 app['SourceCode'] = ''
1788 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1789 if 'name' in apk and apk['name'] != '':
1790 app['Name'] = apk['name']
1792 logging.warning(_('{appid} does not have a name! Using package name instead.')
1793 .format(appid=apk['packageName']))
1794 app['Name'] = apk['packageName']
1795 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1796 yaml.dump(app, f, default_flow_style=False)
1797 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1802 start_timestamp = time.gmtime()
1807 global config, options
1809 # Parse command line...
1810 parser = ArgumentParser()
1811 common.setup_global_opts(parser)
1812 parser.add_argument("--create-key", action="store_true", default=False,
1813 help=_("Add a repo signing key to an unsigned repo"))
1814 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1815 help=_("Add skeleton metadata files for APKs that are missing them"))
1816 parser.add_argument("--delete-unknown", action="store_true", default=False,
1817 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1818 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1819 help=_("Report on build data status"))
1820 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1821 help=_("Interactively ask about things that need updating."))
1822 parser.add_argument("-I", "--icons", action="store_true", default=False,
1823 help=_("Resize all the icons exceeding the max pixel size and exit"))
1824 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1825 help=_("Specify editor to use in interactive mode. Default " +
1826 "is {path}").format(path='/etc/alternatives/editor'))
1827 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1828 help=_("Update the wiki"))
1829 parser.add_argument("--pretty", action="store_true", default=False,
1830 help=_("Produce human-readable XML/JSON for index files"))
1831 parser.add_argument("--clean", action="store_true", default=False,
1832 help=_("Clean update - don't uses caches, reprocess all APKs"))
1833 parser.add_argument("--nosign", action="store_true", default=False,
1834 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1835 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1836 help=_("Use date from APK instead of current time for newly added APKs"))
1837 parser.add_argument("--rename-apks", action="store_true", default=False,
1838 help=_("Rename APK files that do not match package.name_123.apk"))
1839 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1840 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1841 metadata.add_metadata_arguments(parser)
1842 options = parser.parse_args()
1843 metadata.warnings_action = options.W
1845 config = common.read_config(options)
1847 if not ('jarsigner' in config and 'keytool' in config):
1848 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1851 if config['archive_older'] != 0:
1852 repodirs.append('archive')
1853 if not os.path.exists('archive'):
1857 resize_all_icons(repodirs)
1860 if options.rename_apks:
1861 options.clean = True
1863 # check that icons exist now, rather than fail at the end of `fdroid update`
1864 for k in ['repo_icon', 'archive_icon']:
1866 if not os.path.exists(config[k]):
1867 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1868 .format(name=k, path=config[k]))
1871 # if the user asks to create a keystore, do it now, reusing whatever it can
1872 if options.create_key:
1873 if os.path.exists(config['keystore']):
1874 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1875 logging.critical("\t'" + config['keystore'] + "'")
1878 if 'repo_keyalias' not in config:
1879 config['repo_keyalias'] = socket.getfqdn()
1880 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1881 if 'keydname' not in config:
1882 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1883 common.write_to_config(config, 'keydname', config['keydname'])
1884 if 'keystore' not in config:
1885 config['keystore'] = common.default_config['keystore']
1886 common.write_to_config(config, 'keystore', config['keystore'])
1888 password = common.genpassword()
1889 if 'keystorepass' not in config:
1890 config['keystorepass'] = password
1891 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1892 if 'keypass' not in config:
1893 config['keypass'] = password
1894 common.write_to_config(config, 'keypass', config['keypass'])
1895 common.genkeystore(config)
1898 apps = metadata.read_metadata()
1900 # Generate a list of categories...
1902 for app in apps.values():
1903 categories.update(app.Categories)
1905 # Read known apks data (will be updated and written back when we've finished)
1906 knownapks = common.KnownApks()
1909 apkcache = get_cache()
1911 # Delete builds for disabled apps
1912 delete_disabled_builds(apps, apkcache, repodirs)
1914 # Scan all apks in the main repo
1915 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1917 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1918 options.use_date_from_apk)
1919 cachechanged = cachechanged or fcachechanged
1922 if apk['packageName'] not in apps:
1923 if options.create_metadata:
1924 create_metadata_from_template(apk)
1925 apps = metadata.read_metadata()
1927 msg = _("{apkfilename} ({appid}) has no metadata!") \
1928 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1929 if options.delete_unknown:
1930 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1931 .format(apkfilename=apk['apkName']))
1932 rmf = os.path.join(repodirs[0], apk['apkName'])
1933 if not os.path.exists(rmf):
1934 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1938 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1940 copy_triple_t_store_metadata(apps)
1941 insert_obbs(repodirs[0], apps, apks)
1942 insert_localized_app_metadata(apps)
1943 translate_per_build_anti_features(apps, apks)
1945 # Scan the archive repo for apks as well
1946 if len(repodirs) > 1:
1947 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1953 # Apply information from latest apks to the application and update dates
1954 apply_info_from_latest_apk(apps, apks + archapks)
1956 # Sort the app list by name, then the web site doesn't have to by default.
1957 # (we had to wait until we'd scanned the apks to do this, because mostly the
1958 # name comes from there!)
1959 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1961 # APKs are placed into multiple repos based on the app package, providing
1962 # per-app subscription feeds for nightly builds and things like it
1963 if config['per_app_repos']:
1964 add_apks_to_per_app_repos(repodirs[0], apks)
1965 for appid, app in apps.items():
1966 repodir = os.path.join(appid, 'fdroid', 'repo')
1968 appdict[appid] = app
1969 if os.path.isdir(repodir):
1970 index.make(appdict, [appid], apks, repodir, False)
1972 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1975 if len(repodirs) > 1:
1976 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1978 # Make the index for the main repo...
1979 index.make(apps, sortedids, apks, repodirs[0], False)
1980 make_categories_txt(repodirs[0], categories)
1982 # If there's an archive repo, make the index for it. We already scanned it
1984 if len(repodirs) > 1:
1985 index.make(apps, sortedids, archapks, repodirs[1], True)
1987 git_remote = config.get('binary_transparency_remote')
1988 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1990 btlog.make_binary_transparency_log(repodirs)
1992 if config['update_stats']:
1993 # Update known apks info...
1994 knownapks.writeifchanged()
1996 # Generate latest apps data for widget
1997 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1999 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
2001 appid = line.rstrip()
2002 data += appid + "\t"
2004 data += app.Name + "\t"
2005 if app.icon is not None:
2006 data += app.icon + "\t"
2007 data += app.License + "\n"
2008 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2012 write_cache(apkcache)
2014 # Update the wiki...
2016 update_wiki(apps, sortedids, apks + archapks)
2018 logging.info(_("Finished"))
2021 if __name__ == "__main__":