3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
38 from PIL import Image, PngImagePlugin
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
90 def dpi_to_px(density):
91 return (int(density) * 48) / 160
95 return (int(px) * 160) / 48
98 def get_icon_dir(repodir, density):
100 return os.path.join(repodir, "icons")
101 return os.path.join(repodir, "icons-%s" % density)
104 def get_icon_dirs(repodir):
105 for density in screen_densities:
106 yield get_icon_dir(repodir, density)
109 def get_all_icon_dirs(repodir):
110 for density in all_screen_densities:
111 yield get_icon_dir(repodir, density)
114 def update_wiki(apps, sortedids, apks):
117 :param apps: fully populated list of all applications
118 :param apks: all apks, except...
120 logging.info("Updating wiki")
122 wikiredircat = 'App Redirects'
124 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
125 path=config['wiki_path'])
126 site.login(config['wiki_user'], config['wiki_password'])
128 generated_redirects = {}
130 for appid in sortedids:
131 app = metadata.App(apps[appid])
135 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
137 for af in sorted(app.AntiFeatures):
138 wikidata += '{{AntiFeature|' + af + '}}\n'
143 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
146 app.added.strftime('%Y-%m-%d') if app.added else '',
147 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
163 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
165 wikidata += app.Summary
166 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
168 wikidata += "=Description=\n"
169 wikidata += metadata.description_wiki(app.Description) + "\n"
171 wikidata += "=Maintainer Notes=\n"
172 if app.MaintainerNotes:
173 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
174 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
176 # Get a list of all packages for this application...
178 gotcurrentver = False
182 if apk['packageName'] == appid:
183 if str(apk['versionCode']) == app.CurrentVersionCode:
186 # Include ones we can't build, as a special case...
187 for build in app.builds:
189 if build.versionCode == app.CurrentVersionCode:
191 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
192 apklist.append({'versionCode': int(build.versionCode),
193 'versionName': build.versionName,
194 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
199 if apk['versionCode'] == int(build.versionCode):
204 apklist.append({'versionCode': int(build.versionCode),
205 'versionName': build.versionName,
206 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
208 if app.CurrentVersionCode == '0':
210 # Sort with most recent first...
211 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
213 wikidata += "=Versions=\n"
214 if len(apklist) == 0:
215 wikidata += "We currently have no versions of this app available."
216 elif not gotcurrentver:
217 wikidata += "We don't have the current version of this app."
219 wikidata += "We have the current version of this app."
220 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
221 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
222 if len(app.NoSourceSince) > 0:
223 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
224 if len(app.CurrentVersion) > 0:
225 wikidata += "The current (recommended) version is " + app.CurrentVersion
226 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
229 wikidata += "==" + apk['versionName'] + "==\n"
231 if 'buildproblem' in apk:
232 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
235 wikidata += "This version is built and signed by "
237 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
239 wikidata += "the original developer.\n\n"
240 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
242 wikidata += '\n[[Category:' + wikicat + ']]\n'
243 if len(app.NoSourceSince) > 0:
244 wikidata += '\n[[Category:Apps missing source code]]\n'
245 if validapks == 0 and not app.Disabled:
246 wikidata += '\n[[Category:Apps with no packages]]\n'
247 if cantupdate and not app.Disabled:
248 wikidata += "\n[[Category:Apps we cannot update]]\n"
249 if buildfails and not app.Disabled:
250 wikidata += "\n[[Category:Apps with failing builds]]\n"
251 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
252 wikidata += '\n[[Category:Apps to Update]]\n'
254 wikidata += '\n[[Category:Apps that are disabled]]\n'
255 if app.UpdateCheckMode == 'None' and not app.Disabled:
256 wikidata += '\n[[Category:Apps with no update check]]\n'
257 for appcat in app.Categories:
258 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
260 # We can't have underscores in the page name, even if they're in
261 # the package ID, because MediaWiki messes with them...
262 pagename = appid.replace('_', ' ')
264 # Drop a trailing newline, because mediawiki is going to drop it anyway
265 # and it we don't we'll think the page has changed when it hasn't...
266 if wikidata.endswith('\n'):
267 wikidata = wikidata[:-1]
269 generated_pages[pagename] = wikidata
271 # Make a redirect from the name to the ID too, unless there's
272 # already an existing page with the name and it isn't a redirect.
274 apppagename = app.Name.replace('_', ' ')
275 apppagename = apppagename.replace('{', '')
276 apppagename = apppagename.replace('}', ' ')
277 apppagename = apppagename.replace(':', ' ')
278 apppagename = apppagename.replace('[', ' ')
279 apppagename = apppagename.replace(']', ' ')
280 # Drop double spaces caused mostly by replacing ':' above
281 apppagename = apppagename.replace(' ', ' ')
282 for expagename in site.allpages(prefix=apppagename,
283 filterredir='nonredirects',
285 if expagename == apppagename:
287 # Another reason not to make the redirect page is if the app name
288 # is the same as it's ID, because that will overwrite the real page
289 # with an redirect to itself! (Although it seems like an odd
290 # scenario this happens a lot, e.g. where there is metadata but no
291 # builds or binaries to extract a name from.
292 if apppagename == pagename:
295 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
297 for tcat, genp in [(wikicat, generated_pages),
298 (wikiredircat, generated_redirects)]:
299 catpages = site.Pages['Category:' + tcat]
301 for page in catpages:
302 existingpages.append(page.name)
303 if page.name in genp:
304 pagetxt = page.edit()
305 if pagetxt != genp[page.name]:
306 logging.debug("Updating modified page " + page.name)
307 page.save(genp[page.name], summary='Auto-updated')
309 logging.debug("Page " + page.name + " is unchanged")
311 logging.warn("Deleting page " + page.name)
312 page.delete('No longer published')
313 for pagename, text in genp.items():
314 logging.debug("Checking " + pagename)
315 if pagename not in existingpages:
316 logging.debug("Creating page " + pagename)
318 newpage = site.Pages[pagename]
319 newpage.save(text, summary='Auto-created')
320 except Exception as e:
321 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
323 # Purge server cache to ensure counts are up to date
324 site.Pages['Repository Maintenance'].purge()
326 # Write a page with the last build log for this version code
327 wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
328 newpage = site.Pages[wiki_page_path]
330 txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
331 txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
332 txt += "* completed at " + common.get_wiki_timestamp() + '\n'
334 txt += common.get_android_tools_version_log()
335 newpage.save(txt, summary='Run log')
336 newpage = site.Pages['update']
337 newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
340 def delete_disabled_builds(apps, apkcache, repodirs):
341 """Delete disabled build outputs.
343 :param apps: list of all applications, as per metadata.read_metadata
344 :param apkcache: current apk cache information
345 :param repodirs: the repo directories to process
347 for appid, app in apps.items():
348 for build in app['builds']:
349 if not build.disable:
351 apkfilename = common.get_release_filename(app, build)
352 iconfilename = "%s.%s.png" % (
355 for repodir in repodirs:
357 os.path.join(repodir, apkfilename),
358 os.path.join(repodir, apkfilename + '.asc'),
359 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
361 for density in all_screen_densities:
362 repo_dir = get_icon_dir(repodir, density)
363 files.append(os.path.join(repo_dir, iconfilename))
366 if os.path.exists(f):
367 logging.info("Deleting disabled build output " + f)
369 if apkfilename in apkcache:
370 del apkcache[apkfilename]
373 def resize_icon(iconpath, density):
375 if not os.path.isfile(iconpath):
380 fp = open(iconpath, 'rb')
382 size = dpi_to_px(density)
384 if any(length > size for length in im.size):
386 im.thumbnail((size, size), Image.ANTIALIAS)
387 logging.debug("%s was too large at %s - new size is %s" % (
388 iconpath, oldsize, im.size))
389 im.save(iconpath, "PNG", optimize=True,
390 pnginfo=BLANK_PNG_INFO, icc_profile=None)
392 except Exception as e:
393 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
400 def resize_all_icons(repodirs):
401 """Resize all icons that exceed the max size
403 :param repodirs: the repo directories to process
405 for repodir in repodirs:
406 for density in screen_densities:
407 icon_dir = get_icon_dir(repodir, density)
408 icon_glob = os.path.join(icon_dir, '*.png')
409 for iconpath in glob.glob(icon_glob):
410 resize_icon(iconpath, density)
414 """ Get the signing certificate of an apk. To get the same md5 has that
415 Android gets, we encode the .RSA certificate in a specific format and pass
416 it hex-encoded to the md5 digest algorithm.
418 :param apkpath: path to the apk
419 :returns: A string containing the md5 of the signature of the apk or None
420 if an error occurred.
423 with zipfile.ZipFile(apkpath, 'r') as apk:
424 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
427 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
430 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
433 cert = apk.read(certs[0])
435 cert_encoded = common.get_certificate(cert)
437 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
440 def get_cache_file():
441 return os.path.join('tmp', 'apkcache')
445 """Get the cached dict of the APK index
447 Gather information about all the apk files in the repo directory,
448 using cached data if possible. Some of the index operations take a
449 long time, like calculating the SHA-256 and verifying the APK
452 The cache is invalidated if the metadata version is different, or
453 the 'allow_disabled_algorithms' config/option is different. In
454 those cases, there is no easy way to know what has changed from
455 the cache, so just rerun the whole thing.
460 apkcachefile = get_cache_file()
461 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
462 if not options.clean and os.path.exists(apkcachefile):
463 with open(apkcachefile, 'rb') as cf:
464 apkcache = pickle.load(cf, encoding='utf-8')
465 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
466 or apkcache.get('allow_disabled_algorithms') != ada:
471 apkcache["METADATA_VERSION"] = METADATA_VERSION
472 apkcache['allow_disabled_algorithms'] = ada
477 def write_cache(apkcache):
478 apkcachefile = get_cache_file()
479 cache_path = os.path.dirname(apkcachefile)
480 if not os.path.exists(cache_path):
481 os.makedirs(cache_path)
482 with open(apkcachefile, 'wb') as cf:
483 pickle.dump(apkcache, cf)
486 def get_icon_bytes(apkzip, iconsrc):
487 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
489 return apkzip.read(iconsrc)
491 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
494 def sha256sum(filename):
495 '''Calculate the sha256 of the given file'''
496 sha = hashlib.sha256()
497 with open(filename, 'rb') as f:
503 return sha.hexdigest()
506 def has_known_vulnerability(filename):
507 """checks for known vulnerabilities in the APK
509 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
510 version. Google also enforces this:
511 https://support.google.com/faqs/answer/6376725?hl=en
513 Checks whether there are more than one classes.dex or AndroidManifest.xml
514 files, which is invalid and an essential part of the "Master Key" attack.
515 http://www.saurik.com/id/17
517 Janus is similar to Master Key but is perhaps easier to scan for.
518 https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
523 # statically load this pattern
524 if not hasattr(has_known_vulnerability, "pattern"):
525 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
527 with open(filename.encode(), 'rb') as fp:
529 if first4 != b'\x50\x4b\x03\x04':
530 raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
531 .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
532 + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
535 with zipfile.ZipFile(filename) as zf:
536 for name in zf.namelist():
537 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
540 chunk = lib.read(4096)
543 m = has_known_vulnerability.pattern.search(chunk)
545 version = m.group(1).decode('ascii')
546 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
547 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
548 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
549 logging.debug(_('"{path}" contains recent {name} ({version})')
550 .format(path=filename, name=name, version=version))
552 logging.warning(_('"{path}" contains outdated {name} ({version})')
553 .format(path=filename, name=name, version=version))
556 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
557 if name in files_in_apk:
558 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
559 .format(apkfilename=filename, name=name))
561 files_in_apk.add(name)
565 def insert_obbs(repodir, apps, apks):
566 """Scans the .obb files in a given repo directory and adds them to the
567 relevant APK instances. OBB files have versionCodes like APK
568 files, and they are loosely associated. If there is an OBB file
569 present, then any APK with the same or higher versionCode will use
570 that OBB file. There are two OBB types: main and patch, each APK
571 can only have only have one of each.
573 https://developer.android.com/google/play/expansion-files.html
575 :param repodir: repo directory to scan
576 :param apps: list of current, valid apps
577 :param apks: current information on all APKs
581 def obbWarnDelete(f, msg):
582 logging.warning(msg + ' ' + f)
583 if options.delete_unknown:
584 logging.error(_("Deleting unknown file: {path}").format(path=f))
588 java_Integer_MIN_VALUE = -pow(2, 31)
589 currentPackageNames = apps.keys()
590 for f in glob.glob(os.path.join(repodir, '*.obb')):
591 obbfile = os.path.basename(f)
592 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
593 chunks = obbfile.split('.')
594 if chunks[0] != 'main' and chunks[0] != 'patch':
595 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
597 if not re.match(r'^-?[0-9]+$', chunks[1]):
598 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
599 .format(name=chunks[0]))
601 versionCode = int(chunks[1])
602 packagename = ".".join(chunks[2:-1])
604 highestVersionCode = java_Integer_MIN_VALUE
605 if packagename not in currentPackageNames:
606 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
609 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
610 highestVersionCode = apk['versionCode']
611 if versionCode > highestVersionCode:
612 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
613 .format(integer=str(versionCode)))
615 obbsha256 = sha256sum(f)
616 obbs.append((packagename, versionCode, obbfile, obbsha256))
619 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
620 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
621 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
622 apk['obbMainFile'] = obbfile
623 apk['obbMainFileSha256'] = obbsha256
624 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
625 apk['obbPatchFile'] = obbfile
626 apk['obbPatchFileSha256'] = obbsha256
627 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
631 def translate_per_build_anti_features(apps, apks):
632 """Grab the anti-features list from the build metadata
634 For most Anti-Features, they are really most applicable per-APK,
635 not for an app. An app can fix a vulnerability, add/remove
636 tracking, etc. This reads the 'antifeatures' list from the Build
637 entries in the fdroiddata metadata file, then transforms it into
638 the 'antiFeatures' list of unique items for the index.
640 The field key is all lower case in the metadata file to match the
641 rest of the Build fields. It is 'antiFeatures' camel case in the
642 implementation, index, and fdroidclient since it is translated
643 from the build 'antifeatures' field, not directly included.
647 antiFeatures = dict()
648 for packageName, app in apps.items():
650 for build in app['builds']:
651 afl = build.get('antifeatures')
653 d[int(build.versionCode)] = afl
655 antiFeatures[packageName] = d
658 d = antiFeatures.get(apk['packageName'])
660 afl = d.get(apk['versionCode'])
662 apk['antiFeatures'].update(afl)
665 def _get_localized_dict(app, locale):
666 '''get the dict to add localized store metadata to'''
667 if 'localized' not in app:
668 app['localized'] = collections.OrderedDict()
669 if locale not in app['localized']:
670 app['localized'][locale] = collections.OrderedDict()
671 return app['localized'][locale]
674 def _set_localized_text_entry(app, locale, key, f):
675 limit = config['char_limits'][key]
676 localized = _get_localized_dict(app, locale)
678 text = fp.read()[:limit]
680 localized[key] = text
683 def _set_author_entry(app, key, f):
684 limit = config['char_limits']['author']
686 text = fp.read()[:limit]
691 def _strip_and_copy_image(inpath, outpath):
692 """Remove any metadata from image and copy it to new path
694 Sadly, image metadata like EXIF can be used to exploit devices.
695 It is not used at all in the F-Droid ecosystem, so its much safer
696 just to remove it entirely.
700 extension = common.get_extension(inpath)[1]
701 if os.path.isdir(outpath):
702 outpath = os.path.join(outpath, os.path.basename(inpath))
703 if extension == 'png':
704 with open(inpath, 'rb') as fp:
705 in_image = Image.open(fp)
706 in_image.save(outpath, "PNG", optimize=True,
707 pnginfo=BLANK_PNG_INFO, icc_profile=None)
708 elif extension == 'jpg' or extension == 'jpeg':
709 with open(inpath, 'rb') as fp:
710 in_image = Image.open(fp)
711 data = list(in_image.getdata())
712 out_image = Image.new(in_image.mode, in_image.size)
713 out_image.putdata(data)
714 out_image.save(outpath, "JPEG", optimize=True)
716 raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
717 .format(extension=extension))
720 def copy_triple_t_store_metadata(apps):
721 """Include store metadata from the app's source repo
723 The Triple-T Gradle Play Publisher is a plugin that has a standard
724 file layout for all of the metadata and graphics that the Google
725 Play Store accepts. Since F-Droid has the git repo, it can just
726 pluck those files directly. This method reads any text files into
727 the app dict, then copies any graphics into the fdroid repo
730 This needs to be run before insert_localized_app_metadata() so that
731 the graphics files that are copied into the fdroid repo get
734 https://github.com/Triple-T/gradle-play-publisher#upload-images
735 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
739 if not os.path.isdir('build'):
740 return # nothing to do
742 for packageName, app in apps.items():
743 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
744 logging.debug('Triple-T Gradle Play Publisher: ' + d)
745 for root, dirs, files in os.walk(d):
746 segments = root.split('/')
747 locale = segments[-2]
749 if f == 'fulldescription':
750 _set_localized_text_entry(app, locale, 'description',
751 os.path.join(root, f))
753 elif f == 'shortdescription':
754 _set_localized_text_entry(app, locale, 'summary',
755 os.path.join(root, f))
758 _set_localized_text_entry(app, locale, 'name',
759 os.path.join(root, f))
762 _set_localized_text_entry(app, locale, 'video',
763 os.path.join(root, f))
765 elif f == 'whatsnew':
766 _set_localized_text_entry(app, segments[-1], 'whatsNew',
767 os.path.join(root, f))
769 elif f == 'contactEmail':
770 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
772 elif f == 'contactPhone':
773 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
775 elif f == 'contactWebsite':
776 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
779 base, extension = common.get_extension(f)
780 dirname = os.path.basename(root)
781 if extension in ALLOWED_EXTENSIONS \
782 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
783 if segments[-2] == 'listing':
784 locale = segments[-3]
786 locale = segments[-2]
787 destdir = os.path.join('repo', packageName, locale, dirname)
788 os.makedirs(destdir, mode=0o755, exist_ok=True)
789 sourcefile = os.path.join(root, f)
790 destfile = os.path.join(destdir, os.path.basename(f))
791 logging.debug('copying ' + sourcefile + ' ' + destfile)
792 _strip_and_copy_image(sourcefile, destfile)
795 def insert_localized_app_metadata(apps):
796 """scans standard locations for graphics and localized text
798 Scans for localized description files, store graphics, and
799 screenshot PNG files in statically defined screenshots directory
800 and adds them to the app metadata. The screenshots and graphic
801 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
802 and must be in the following layout:
803 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
805 repo/packageName/locale/featureGraphic.png
806 repo/packageName/locale/phoneScreenshots/1.png
807 repo/packageName/locale/phoneScreenshots/2.png
809 The changelog files must be text files named with the versionCode
810 ending with ".txt" and must be in the following layout:
811 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
813 repo/packageName/locale/changelogs/12345.txt
815 This will scan the each app's source repo then the metadata/ dir
816 for these standard locations of changelog files. If it finds
817 them, they will be added to the dict of all packages, with the
818 versions in the metadata/ folder taking precendence over the what
819 is in the app's source repo.
821 Where "packageName" is the app's packageName and "locale" is the locale
822 of the graphics, e.g. what language they are in, using the IETF RFC5646
823 format (en-US, fr-CA, es-MX, etc).
825 This will also scan the app's git for a fastlane folder, and the
826 metadata/ folder and the apps' source repos for standard locations
827 of graphic and screenshot files. If it finds them, it will copy
828 them into the repo. The fastlane files follow this pattern:
829 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
833 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
834 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
835 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
836 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
838 for srcd in sorted(sourcedirs):
839 if not os.path.isdir(srcd):
841 for root, dirs, files in os.walk(srcd):
842 segments = root.split('/')
843 packageName = segments[1]
844 if packageName not in apps:
845 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
847 locale = segments[-1]
848 destdir = os.path.join('repo', packageName, locale)
850 # flavours specified in build receipt
852 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
853 and 'gradle' in apps[packageName].builds[-1]:
854 build_flavours = apps[packageName].builds[-1].gradle
856 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
857 logging.debug("ignoring due to wrong flavour")
861 if f in ('description.txt', 'full_description.txt'):
862 _set_localized_text_entry(apps[packageName], locale, 'description',
863 os.path.join(root, f))
865 elif f in ('summary.txt', 'short_description.txt'):
866 _set_localized_text_entry(apps[packageName], locale, 'summary',
867 os.path.join(root, f))
869 elif f in ('name.txt', 'title.txt'):
870 _set_localized_text_entry(apps[packageName], locale, 'name',
871 os.path.join(root, f))
873 elif f == 'video.txt':
874 _set_localized_text_entry(apps[packageName], locale, 'video',
875 os.path.join(root, f))
877 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
878 locale = segments[-2]
879 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
880 os.path.join(root, f))
883 base, extension = common.get_extension(f)
884 if locale == 'images':
885 locale = segments[-2]
886 destdir = os.path.join('repo', packageName, locale)
887 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
888 os.makedirs(destdir, mode=0o755, exist_ok=True)
889 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
890 _strip_and_copy_image(os.path.join(root, f), destdir)
892 if d in SCREENSHOT_DIRS:
893 if locale == 'images':
894 locale = segments[-2]
895 destdir = os.path.join('repo', packageName, locale)
896 for f in glob.glob(os.path.join(root, d, '*.*')):
897 _ignored, extension = common.get_extension(f)
898 if extension in ALLOWED_EXTENSIONS:
899 screenshotdestdir = os.path.join(destdir, d)
900 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
901 logging.debug('copying ' + f + ' ' + screenshotdestdir)
902 _strip_and_copy_image(f, screenshotdestdir)
904 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
906 if not os.path.isdir(d):
908 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
909 if not os.path.isfile(f):
911 segments = f.split('/')
912 packageName = segments[1]
914 screenshotdir = segments[3]
915 filename = os.path.basename(f)
916 base, extension = common.get_extension(filename)
918 if packageName not in apps:
919 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
920 .format(path=filename, name=packageName))
922 graphics = _get_localized_dict(apps[packageName], locale)
924 if extension not in ALLOWED_EXTENSIONS:
925 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
926 elif base in GRAPHIC_NAMES:
927 # there can only be zero or one of these per locale
928 graphics[base] = filename
929 elif screenshotdir in SCREENSHOT_DIRS:
930 # there can any number of these per locale
931 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
932 if screenshotdir not in graphics:
933 graphics[screenshotdir] = []
934 graphics[screenshotdir].append(filename)
936 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
940 """Scan a repo for all files with an extension except APK/OBB
942 :param apkcache: current cached info about all repo files
943 :param repodir: repo directory to scan
944 :param knownapks: list of all known files, as per metadata.read_metadata
945 :param use_date_from_file: use date from file (instead of current date)
946 for newly added files
951 repodir = repodir.encode('utf-8')
952 for name in os.listdir(repodir):
953 file_extension = common.get_file_extension(name)
954 if file_extension == 'apk' or file_extension == 'obb':
956 filename = os.path.join(repodir, name)
957 name_utf8 = name.decode('utf-8')
958 if filename.endswith(b'_src.tar.gz'):
959 logging.debug(_('skipping source tarball: {path}')
960 .format(path=filename.decode('utf-8')))
962 if not common.is_repo_file(filename):
964 stat = os.stat(filename)
965 if stat.st_size == 0:
966 raise FDroidException(_('{path} is zero size!')
967 .format(path=filename))
969 shasum = sha256sum(filename)
972 repo_file = apkcache[name]
973 # added time is cached as tuple but used here as datetime instance
974 if 'added' in repo_file:
975 a = repo_file['added']
976 if isinstance(a, datetime):
977 repo_file['added'] = a
979 repo_file['added'] = datetime(*a[:6])
980 if repo_file.get('hash') == shasum:
981 logging.debug(_("Reading {apkfilename} from cache")
982 .format(apkfilename=name_utf8))
985 logging.debug(_("Ignoring stale cache data for {apkfilename}")
986 .format(apkfilename=name_utf8))
989 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
990 repo_file = collections.OrderedDict()
991 repo_file['name'] = os.path.splitext(name_utf8)[0]
992 # TODO rename apkname globally to something more generic
993 repo_file['apkName'] = name_utf8
994 repo_file['hash'] = shasum
995 repo_file['hashType'] = 'sha256'
996 repo_file['versionCode'] = 0
997 repo_file['versionName'] = shasum
998 # the static ID is the SHA256 unless it is set in the metadata
999 repo_file['packageName'] = shasum
1001 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1003 repo_file['packageName'] = m.group(1)
1004 repo_file['versionCode'] = int(m.group(2))
1005 srcfilename = name + b'_src.tar.gz'
1006 if os.path.exists(os.path.join(repodir, srcfilename)):
1007 repo_file['srcname'] = srcfilename.decode('utf-8')
1008 repo_file['size'] = stat.st_size
1010 apkcache[name] = repo_file
1013 if use_date_from_file:
1014 timestamp = stat.st_ctime
1015 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1017 default_date_param = None
1019 # Record in knownapks, getting the added date at the same time..
1020 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021 default_date=default_date_param)
1023 repo_file['added'] = added
1025 repo_files.append(repo_file)
1027 return repo_files, cachechanged
1030 def scan_apk(apk_file):
1032 Scans an APK file and returns dictionary with metadata of the APK.
1034 Attention: This does *not* verify that the APK signature is correct.
1036 :param apk_file: The (ideally absolute) path to the APK file
1037 :raises BuildException
1038 :return A dict containing APK metadata
1041 'hash': sha256sum(apk_file),
1042 'hashType': 'sha256',
1043 'uses-permission': [],
1044 'uses-permission-sdk-23': [],
1048 'antiFeatures': set(),
1051 if SdkToolsPopen(['aapt', 'version'], output=False):
1052 scan_apk_aapt(apk, apk_file)
1054 scan_apk_androguard(apk, apk_file)
1056 # Get the signature, or rather the signing key fingerprints
1057 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058 apk['sig'] = getsig(apk_file)
1060 raise BuildException("Failed to get apk signature")
1061 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1063 if not apk.get('signer'):
1064 raise BuildException("Failed to get apk signing key fingerprint")
1066 # Get size of the APK
1067 apk['size'] = os.path.getsize(apk_file)
1069 if 'minSdkVersion' not in apk:
1070 logging.warning("No SDK version information found in {0}".format(apk_file))
1071 apk['minSdkVersion'] = 1
1073 # Check for known vulnerabilities
1074 if has_known_vulnerability(apk_file):
1075 apk['antiFeatures'].add('KnownVuln')
1080 def scan_apk_aapt(apk, apkfile):
1081 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1082 if p.returncode != 0:
1083 if options.delete_unknown:
1084 if os.path.exists(apkfile):
1085 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1088 logging.error("Could not find {0} to remove it".format(apkfile))
1090 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1091 raise BuildException(_("Invalid APK"))
1092 for line in p.output.splitlines():
1093 if line.startswith("package:"):
1095 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1096 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1097 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1098 except Exception as e:
1099 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1100 elif line.startswith("application:"):
1101 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1102 # Keep path to non-dpi icon in case we need it
1103 match = re.match(APK_ICON_PAT_NODPI, line)
1105 apk['icons_src']['-1'] = match.group(1)
1106 elif line.startswith("launchable-activity:"):
1107 # Only use launchable-activity as fallback to application
1109 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1110 if '-1' not in apk['icons_src']:
1111 match = re.match(APK_ICON_PAT_NODPI, line)
1113 apk['icons_src']['-1'] = match.group(1)
1114 elif line.startswith("application-icon-"):
1115 match = re.match(APK_ICON_PAT, line)
1117 density = match.group(1)
1118 path = match.group(2)
1119 apk['icons_src'][density] = path
1120 elif line.startswith("sdkVersion:"):
1121 m = re.match(APK_SDK_VERSION_PAT, line)
1123 logging.error(line.replace('sdkVersion:', '')
1124 + ' is not a valid minSdkVersion!')
1126 apk['minSdkVersion'] = m.group(1)
1127 # if target not set, default to min
1128 if 'targetSdkVersion' not in apk:
1129 apk['targetSdkVersion'] = m.group(1)
1130 elif line.startswith("targetSdkVersion:"):
1131 m = re.match(APK_SDK_VERSION_PAT, line)
1133 logging.error(line.replace('targetSdkVersion:', '')
1134 + ' is not a valid targetSdkVersion!')
1136 apk['targetSdkVersion'] = m.group(1)
1137 elif line.startswith("maxSdkVersion:"):
1138 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1139 elif line.startswith("native-code:"):
1140 apk['nativecode'] = []
1141 for arch in line[13:].split(' '):
1142 apk['nativecode'].append(arch[1:-1])
1143 elif line.startswith('uses-permission:'):
1144 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1145 if perm_match['maxSdkVersion']:
1146 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1147 permission = UsesPermission(
1149 perm_match['maxSdkVersion']
1152 apk['uses-permission'].append(permission)
1153 elif line.startswith('uses-permission-sdk-23:'):
1154 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1155 if perm_match['maxSdkVersion']:
1156 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1157 permission_sdk_23 = UsesPermissionSdk23(
1159 perm_match['maxSdkVersion']
1162 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1164 elif line.startswith('uses-feature:'):
1165 feature = re.match(APK_FEATURE_PAT, line).group(1)
1166 # Filter out this, it's only added with the latest SDK tools and
1167 # causes problems for lots of apps.
1168 if feature != "android.hardware.screen.portrait" \
1169 and feature != "android.hardware.screen.landscape":
1170 if feature.startswith("android.feature."):
1171 feature = feature[16:]
1172 apk['features'].add(feature)
1175 def scan_apk_androguard(apk, apkfile):
1177 from androguard.core.bytecodes.apk import APK
1178 apkobject = APK(apkfile)
1179 if apkobject.is_valid_APK():
1180 arsc = apkobject.get_android_resources()
1182 if options.delete_unknown:
1183 if os.path.exists(apkfile):
1184 logging.error(_("Failed to get apk information, deleting {path}")
1185 .format(path=apkfile))
1188 logging.error(_("Could not find {path} to remove it")
1189 .format(path=apkfile))
1191 logging.error(_("Failed to get apk information, skipping {path}")
1192 .format(path=apkfile))
1193 raise BuildException(_("Invalid APK"))
1195 raise FDroidException("androguard library is not installed and aapt not present")
1196 except FileNotFoundError:
1197 logging.error(_("Could not open apk file for analysis"))
1198 raise BuildException(_("Invalid APK"))
1200 apk['packageName'] = apkobject.get_package()
1201 apk['versionCode'] = int(apkobject.get_androidversion_code())
1202 apk['versionName'] = apkobject.get_androidversion_name()
1203 if apk['versionName'][0] == "@":
1204 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1205 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1206 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1207 apk['name'] = apkobject.get_app_name()
1209 if apkobject.get_max_sdk_version() is not None:
1210 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1211 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1212 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1214 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1215 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1217 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1219 for file in apkobject.get_files():
1220 d_re = density_re.match(file)
1222 folder = d_re.group(1).split('-')
1224 resolution = folder[1]
1227 density = screen_resolutions[resolution]
1228 apk['icons_src'][density] = d_re.group(0)
1230 if apk['icons_src'].get('-1') is None:
1231 apk['icons_src']['-1'] = apk['icons_src']['160']
1233 arch_re = re.compile("^lib/(.*)/.*$")
1234 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1236 apk['nativecode'] = []
1237 apk['nativecode'].extend(sorted(list(arch)))
1239 xml = apkobject.get_android_manifest_xml()
1241 for item in xml.getElementsByTagName('uses-permission'):
1242 name = str(item.getAttribute("android:name"))
1243 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1244 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1245 permission = UsesPermission(
1249 apk['uses-permission'].append(permission)
1251 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1252 name = str(item.getAttribute("android:name"))
1253 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1254 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1255 permission_sdk_23 = UsesPermissionSdk23(
1259 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1261 for item in xml.getElementsByTagName('uses-feature'):
1262 feature = str(item.getAttribute("android:name"))
1263 if feature != "android.hardware.screen.portrait" \
1264 and feature != "android.hardware.screen.landscape":
1265 if feature.startswith("android.feature."):
1266 feature = feature[16:]
1267 apk['features'].append(feature)
1270 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1271 allow_disabled_algorithms=False, archive_bad_sig=False):
1272 """Processes the apk with the given filename in the given repo directory.
1274 This also extracts the icons.
1276 :param apkcache: current apk cache information
1277 :param apkfilename: the filename of the apk to scan
1278 :param repodir: repo directory to scan
1279 :param knownapks: known apks info
1280 :param use_date_from_apk: use date from APK (instead of current date)
1281 for newly added APKs
1282 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1283 disabled algorithms in the signature (e.g. MD5)
1284 :param archive_bad_sig: move APKs with a bad signature to the archive
1285 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1286 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1290 apkfile = os.path.join(repodir, apkfilename)
1292 cachechanged = False
1294 if apkfilename in apkcache:
1295 apk = apkcache[apkfilename]
1296 if apk.get('hash') == sha256sum(apkfile):
1297 logging.debug(_("Reading {apkfilename} from cache")
1298 .format(apkfilename=apkfilename))
1301 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1302 .format(apkfilename=apkfilename))
1305 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1308 apk = scan_apk(apkfile)
1309 except BuildException:
1310 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1311 .format(apkfilename=apkfilename))
1312 return True, None, False
1314 # Check for debuggable apks...
1315 if common.isApkAndDebuggable(apkfile):
1316 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1318 if options.rename_apks:
1319 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1320 std_short_name = os.path.join(repodir, n)
1321 if apkfile != std_short_name:
1322 if os.path.exists(std_short_name):
1323 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1324 if apkfile != std_long_name:
1325 if os.path.exists(std_long_name):
1326 dupdir = os.path.join('duplicates', repodir)
1327 if not os.path.isdir(dupdir):
1328 os.makedirs(dupdir, exist_ok=True)
1329 dupfile = os.path.join('duplicates', std_long_name)
1330 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1331 os.rename(apkfile, dupfile)
1332 return True, None, False
1334 os.rename(apkfile, std_long_name)
1335 apkfile = std_long_name
1337 os.rename(apkfile, std_short_name)
1338 apkfile = std_short_name
1339 apkfilename = apkfile[len(repodir) + 1:]
1341 apk['apkName'] = apkfilename
1342 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1343 if os.path.exists(os.path.join(repodir, srcfilename)):
1344 apk['srcname'] = srcfilename
1346 # verify the jar signature is correct, allow deprecated
1347 # algorithms only if the APK is in the archive.
1349 if not common.verify_apk_signature(apkfile):
1350 if repodir == 'archive' or allow_disabled_algorithms:
1351 if common.verify_old_apk_signature(apkfile):
1352 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1360 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1361 .format(apkfilename=apkfilename))
1362 move_apk_between_sections(repodir, 'archive', apk)
1364 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1365 .format(apkfilename=apkfilename))
1366 return True, None, False
1368 apkzip = zipfile.ZipFile(apkfile, 'r')
1370 manifest = apkzip.getinfo('AndroidManifest.xml')
1371 # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1372 if (1980, 0, 0) != manifest.date_time[0:3]:
1374 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1375 except ValueError as e:
1376 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1377 .format(apkfilename=apkfile) + str(e))
1379 # extract icons from APK zip file
1380 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1382 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1384 apkzip.close() # ensure that APK zip file gets closed
1386 # resize existing icons for densities missing in the APK
1387 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1389 if use_date_from_apk and manifest.date_time[1] != 0:
1390 default_date_param = datetime(*manifest.date_time)
1392 default_date_param = None
1394 # Record in known apks, getting the added date at the same time..
1395 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1396 default_date=default_date_param)
1398 apk['added'] = added
1400 apkcache[apkfilename] = apk
1403 return False, apk, cachechanged
1406 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1407 """Processes the apks in the given repo directory.
1409 This also extracts the icons.
1411 :param apkcache: current apk cache information
1412 :param repodir: repo directory to scan
1413 :param knownapks: known apks info
1414 :param use_date_from_apk: use date from APK (instead of current date)
1415 for newly added APKs
1416 :returns: (apks, cachechanged) where apks is a list of apk information,
1417 and cachechanged is True if the apkcache got changed.
1420 cachechanged = False
1422 for icon_dir in get_all_icon_dirs(repodir):
1423 if os.path.exists(icon_dir):
1425 shutil.rmtree(icon_dir)
1426 os.makedirs(icon_dir)
1428 os.makedirs(icon_dir)
1431 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1432 apkfilename = apkfile[len(repodir) + 1:]
1433 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1434 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1435 use_date_from_apk, ada, True)
1439 cachechanged = cachechanged or cachethis
1441 return apks, cachechanged
1444 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1446 Extracts icons from the given APK zip in various densities,
1447 saves them into given repo directory
1448 and stores their names in the APK metadata dictionary.
1450 :param icon_filename: A string representing the icon's file name
1451 :param apk: A populated dictionary containing APK metadata.
1452 Needs to have 'icons_src' key
1453 :param apkzip: An opened zipfile.ZipFile of the APK file
1454 :param repo_dir: The directory of the APK's repository
1455 :return: A list of icon densities that are missing
1457 empty_densities = []
1458 for density in screen_densities:
1459 if density not in apk['icons_src']:
1460 empty_densities.append(density)
1462 icon_src = apk['icons_src'][density]
1463 icon_dir = get_icon_dir(repo_dir, density)
1464 icon_dest = os.path.join(icon_dir, icon_filename)
1466 # Extract the icon files per density
1467 if icon_src.endswith('.xml'):
1468 png = os.path.basename(icon_src)[:-4] + '.png'
1469 for f in apkzip.namelist():
1471 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1472 if m and screen_resolutions[m.group(2)] == density:
1474 if icon_src.endswith('.xml'):
1475 empty_densities.append(density)
1478 with open(icon_dest, 'wb') as f:
1479 f.write(get_icon_bytes(apkzip, icon_src))
1480 apk['icons'][density] = icon_filename
1481 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1482 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1483 del apk['icons_src'][density]
1484 empty_densities.append(density)
1486 if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1487 icon_src = apk['icons_src']['-1']
1488 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1489 with open(icon_path, 'wb') as f:
1490 f.write(get_icon_bytes(apkzip, icon_src))
1493 im = Image.open(icon_path)
1494 dpi = px_to_dpi(im.size[0])
1495 for density in screen_densities:
1496 if density in apk['icons']:
1498 if density == screen_densities[-1] or dpi >= int(density):
1499 apk['icons'][density] = icon_filename
1500 shutil.move(icon_path,
1501 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1502 empty_densities.remove(density)
1504 except Exception as e:
1505 logging.warning(_("Failed reading {path}: {error}")
1506 .format(path=icon_path, error=e))
1508 if im and hasattr(im, 'close'):
1512 apk['icon'] = icon_filename
1514 return empty_densities
1517 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1519 Resize existing icons for densities missing in the APK to ensure all densities are available
1521 :param empty_densities: A list of icon densities that are missing
1522 :param icon_filename: A string representing the icon's file name
1523 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1524 :param repo_dir: The directory of the APK's repository
1526 # First try resizing down to not lose quality
1528 for density in screen_densities:
1529 if density not in empty_densities:
1530 last_density = density
1532 if last_density is None:
1534 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1536 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1537 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1540 fp = open(last_icon_path, 'rb')
1543 size = dpi_to_px(density)
1545 im.thumbnail((size, size), Image.ANTIALIAS)
1546 im.save(icon_path, "PNG", optimize=True,
1547 pnginfo=BLANK_PNG_INFO, icc_profile=None)
1548 empty_densities.remove(density)
1549 except Exception as e:
1550 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1555 # Then just copy from the highest resolution available
1557 for density in reversed(screen_densities):
1558 if density not in empty_densities:
1559 last_density = density
1562 if last_density is None:
1566 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1567 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1569 empty_densities.remove(density)
1571 for density in screen_densities:
1572 icon_dir = get_icon_dir(repo_dir, density)
1573 icon_dest = os.path.join(icon_dir, icon_filename)
1574 resize_icon(icon_dest, density)
1576 # Copy from icons-mdpi to icons since mdpi is the baseline density
1577 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1578 if os.path.isfile(baseline):
1579 apk['icons']['0'] = icon_filename
1580 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1583 def apply_info_from_latest_apk(apps, apks):
1585 Some information from the apks needs to be applied up to the application level.
1586 When doing this, we use the info from the most recent version's apk.
1587 We deal with figuring out when the app was added and last updated at the same time.
1589 for appid, app in apps.items():
1590 bestver = UNSET_VERSION_CODE
1592 if apk['packageName'] == appid:
1593 if apk['versionCode'] > bestver:
1594 bestver = apk['versionCode']
1598 if not app.added or apk['added'] < app.added:
1599 app.added = apk['added']
1600 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1601 app.lastUpdated = apk['added']
1604 logging.debug("Don't know when " + appid + " was added")
1605 if not app.lastUpdated:
1606 logging.debug("Don't know when " + appid + " was last updated")
1608 if bestver == UNSET_VERSION_CODE:
1610 if app.Name is None:
1611 app.Name = app.AutoName or appid
1613 logging.debug("Application " + appid + " has no packages")
1615 if app.Name is None:
1616 app.Name = bestapk['name']
1617 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1618 if app.CurrentVersionCode is None:
1619 app.CurrentVersionCode = str(bestver)
1622 def make_categories_txt(repodir, categories):
1623 '''Write a category list in the repo to allow quick access'''
1625 for cat in sorted(categories):
1626 catdata += cat + '\n'
1627 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1631 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1633 def filter_apk_list_sorted(apk_list):
1635 for apk in apk_list:
1636 if apk['packageName'] == appid:
1639 # Sort the apk list by version code. First is highest/newest.
1640 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1642 for appid, app in apps.items():
1644 if app.ArchivePolicy:
1645 keepversions = int(app.ArchivePolicy[:-9])
1647 keepversions = defaultkeepversions
1649 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1650 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1652 current_app_apks = filter_apk_list_sorted(apks)
1653 if len(current_app_apks) > keepversions:
1654 # Move back the ones we don't want.
1655 for apk in current_app_apks[keepversions:]:
1656 move_apk_between_sections(repodir, archivedir, apk)
1657 archapks.append(apk)
1660 current_app_archapks = filter_apk_list_sorted(archapks)
1661 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1663 # Move forward the ones we want again, except DisableAlgorithm
1664 for apk in current_app_archapks:
1665 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1666 move_apk_between_sections(archivedir, repodir, apk)
1667 archapks.remove(apk)
1670 if kept == keepversions:
1674 def move_apk_between_sections(from_dir, to_dir, apk):
1675 """move an APK from repo to archive or vice versa"""
1677 def _move_file(from_dir, to_dir, filename, ignore_missing):
1678 from_path = os.path.join(from_dir, filename)
1679 if ignore_missing and not os.path.exists(from_path):
1681 to_path = os.path.join(to_dir, filename)
1682 if not os.path.exists(to_dir):
1684 shutil.move(from_path, to_path)
1686 if from_dir == to_dir:
1689 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1690 _move_file(from_dir, to_dir, apk['apkName'], False)
1691 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1692 for density in all_screen_densities:
1693 from_icon_dir = get_icon_dir(from_dir, density)
1694 to_icon_dir = get_icon_dir(to_dir, density)
1695 if density not in apk.get('icons', []):
1697 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1698 if 'srcname' in apk:
1699 _move_file(from_dir, to_dir, apk['srcname'], False)
1702 def add_apks_to_per_app_repos(repodir, apks):
1703 apks_per_app = dict()
1705 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1706 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1707 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1708 apks_per_app[apk['packageName']] = apk
1710 if not os.path.exists(apk['per_app_icons']):
1711 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1712 os.makedirs(apk['per_app_icons'])
1714 apkpath = os.path.join(repodir, apk['apkName'])
1715 shutil.copy(apkpath, apk['per_app_repo'])
1716 apksigpath = apkpath + '.sig'
1717 if os.path.exists(apksigpath):
1718 shutil.copy(apksigpath, apk['per_app_repo'])
1719 apkascpath = apkpath + '.asc'
1720 if os.path.exists(apkascpath):
1721 shutil.copy(apkascpath, apk['per_app_repo'])
1724 def create_metadata_from_template(apk):
1725 '''create a new metadata file using internal or external template
1727 Generate warnings for apk's with no metadata (or create skeleton
1728 metadata files, if requested on the command line). Though the
1729 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1730 since those impose things on the metadata file made from the
1731 template: field sort order, empty field value, formatting, etc.
1735 if os.path.exists('template.yml'):
1736 with open('template.yml') as f:
1738 if 'name' in apk and apk['name'] != '':
1739 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1740 r'\1 ' + apk['name'],
1742 flags=re.IGNORECASE | re.MULTILINE)
1744 logging.warning(_('{appid} does not have a name! Using package name instead.')
1745 .format(appid=apk['packageName']))
1746 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1747 r'\1 ' + apk['packageName'],
1749 flags=re.IGNORECASE | re.MULTILINE)
1750 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1754 app['Categories'] = [os.path.basename(os.getcwd())]
1755 # include some blanks as part of the template
1756 app['AuthorName'] = ''
1759 app['IssueTracker'] = ''
1760 app['SourceCode'] = ''
1761 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1762 if 'name' in apk and apk['name'] != '':
1763 app['Name'] = apk['name']
1765 logging.warning(_('{appid} does not have a name! Using package name instead.')
1766 .format(appid=apk['packageName']))
1767 app['Name'] = apk['packageName']
1768 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1769 yaml.dump(app, f, default_flow_style=False)
1770 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1775 start_timestamp = time.gmtime()
1780 global config, options
1782 # Parse command line...
1783 parser = ArgumentParser()
1784 common.setup_global_opts(parser)
1785 parser.add_argument("--create-key", action="store_true", default=False,
1786 help=_("Add a repo signing key to an unsigned repo"))
1787 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1788 help=_("Add skeleton metadata files for APKs that are missing them"))
1789 parser.add_argument("--delete-unknown", action="store_true", default=False,
1790 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1791 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1792 help=_("Report on build data status"))
1793 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1794 help=_("Interactively ask about things that need updating."))
1795 parser.add_argument("-I", "--icons", action="store_true", default=False,
1796 help=_("Resize all the icons exceeding the max pixel size and exit"))
1797 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1798 help=_("Specify editor to use in interactive mode. Default " +
1799 "is {path}").format(path='/etc/alternatives/editor'))
1800 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1801 help=_("Update the wiki"))
1802 parser.add_argument("--pretty", action="store_true", default=False,
1803 help=_("Produce human-readable XML/JSON for index files"))
1804 parser.add_argument("--clean", action="store_true", default=False,
1805 help=_("Clean update - don't uses caches, reprocess all APKs"))
1806 parser.add_argument("--nosign", action="store_true", default=False,
1807 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1808 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1809 help=_("Use date from APK instead of current time for newly added APKs"))
1810 parser.add_argument("--rename-apks", action="store_true", default=False,
1811 help=_("Rename APK files that do not match package.name_123.apk"))
1812 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1813 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1814 metadata.add_metadata_arguments(parser)
1815 options = parser.parse_args()
1816 metadata.warnings_action = options.W
1818 config = common.read_config(options)
1820 if not ('jarsigner' in config and 'keytool' in config):
1821 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1824 if config['archive_older'] != 0:
1825 repodirs.append('archive')
1826 if not os.path.exists('archive'):
1830 resize_all_icons(repodirs)
1833 if options.rename_apks:
1834 options.clean = True
1836 # check that icons exist now, rather than fail at the end of `fdroid update`
1837 for k in ['repo_icon', 'archive_icon']:
1839 if not os.path.exists(config[k]):
1840 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1841 .format(name=k, path=config[k]))
1844 # if the user asks to create a keystore, do it now, reusing whatever it can
1845 if options.create_key:
1846 if os.path.exists(config['keystore']):
1847 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1848 logging.critical("\t'" + config['keystore'] + "'")
1851 if 'repo_keyalias' not in config:
1852 config['repo_keyalias'] = socket.getfqdn()
1853 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1854 if 'keydname' not in config:
1855 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1856 common.write_to_config(config, 'keydname', config['keydname'])
1857 if 'keystore' not in config:
1858 config['keystore'] = common.default_config['keystore']
1859 common.write_to_config(config, 'keystore', config['keystore'])
1861 password = common.genpassword()
1862 if 'keystorepass' not in config:
1863 config['keystorepass'] = password
1864 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1865 if 'keypass' not in config:
1866 config['keypass'] = password
1867 common.write_to_config(config, 'keypass', config['keypass'])
1868 common.genkeystore(config)
1871 apps = metadata.read_metadata()
1873 # Generate a list of categories...
1875 for app in apps.values():
1876 categories.update(app.Categories)
1878 # Read known apks data (will be updated and written back when we've finished)
1879 knownapks = common.KnownApks()
1882 apkcache = get_cache()
1884 # Delete builds for disabled apps
1885 delete_disabled_builds(apps, apkcache, repodirs)
1887 # Scan all apks in the main repo
1888 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1890 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1891 options.use_date_from_apk)
1892 cachechanged = cachechanged or fcachechanged
1895 if apk['packageName'] not in apps:
1896 if options.create_metadata:
1897 create_metadata_from_template(apk)
1898 apps = metadata.read_metadata()
1900 msg = _("{apkfilename} ({appid}) has no metadata!") \
1901 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1902 if options.delete_unknown:
1903 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1904 .format(apkfilename=apk['apkName']))
1905 rmf = os.path.join(repodirs[0], apk['apkName'])
1906 if not os.path.exists(rmf):
1907 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1911 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1913 copy_triple_t_store_metadata(apps)
1914 insert_obbs(repodirs[0], apps, apks)
1915 insert_localized_app_metadata(apps)
1916 translate_per_build_anti_features(apps, apks)
1918 # Scan the archive repo for apks as well
1919 if len(repodirs) > 1:
1920 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1926 # Apply information from latest apks to the application and update dates
1927 apply_info_from_latest_apk(apps, apks + archapks)
1929 # Sort the app list by name, then the web site doesn't have to by default.
1930 # (we had to wait until we'd scanned the apks to do this, because mostly the
1931 # name comes from there!)
1932 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1934 # APKs are placed into multiple repos based on the app package, providing
1935 # per-app subscription feeds for nightly builds and things like it
1936 if config['per_app_repos']:
1937 add_apks_to_per_app_repos(repodirs[0], apks)
1938 for appid, app in apps.items():
1939 repodir = os.path.join(appid, 'fdroid', 'repo')
1941 appdict[appid] = app
1942 if os.path.isdir(repodir):
1943 index.make(appdict, [appid], apks, repodir, False)
1945 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1948 if len(repodirs) > 1:
1949 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1951 # Make the index for the main repo...
1952 index.make(apps, sortedids, apks, repodirs[0], False)
1953 make_categories_txt(repodirs[0], categories)
1955 # If there's an archive repo, make the index for it. We already scanned it
1957 if len(repodirs) > 1:
1958 index.make(apps, sortedids, archapks, repodirs[1], True)
1960 git_remote = config.get('binary_transparency_remote')
1961 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1963 btlog.make_binary_transparency_log(repodirs)
1965 if config['update_stats']:
1966 # Update known apks info...
1967 knownapks.writeifchanged()
1969 # Generate latest apps data for widget
1970 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1972 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1974 appid = line.rstrip()
1975 data += appid + "\t"
1977 data += app.Name + "\t"
1978 if app.icon is not None:
1979 data += app.icon + "\t"
1980 data += app.License + "\n"
1981 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1985 write_cache(apkcache)
1987 # Update the wiki...
1989 update_wiki(apps, sortedids, apks + archapks)
1991 logging.info(_("Finished"))
1994 if __name__ == "__main__":