3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
38 from PIL import Image, PngImagePlugin
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
90 def dpi_to_px(density):
91 return (int(density) * 48) / 160
95 return (int(px) * 160) / 48
98 def get_icon_dir(repodir, density):
100 return os.path.join(repodir, "icons")
101 return os.path.join(repodir, "icons-%s" % density)
104 def get_icon_dirs(repodir):
105 for density in screen_densities:
106 yield get_icon_dir(repodir, density)
109 def get_all_icon_dirs(repodir):
110 for density in all_screen_densities:
111 yield get_icon_dir(repodir, density)
114 def update_wiki(apps, sortedids, apks):
117 :param apps: fully populated list of all applications
118 :param apks: all apks, except...
120 logging.info("Updating wiki")
122 wikiredircat = 'App Redirects'
124 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
125 path=config['wiki_path'])
126 site.login(config['wiki_user'], config['wiki_password'])
128 generated_redirects = {}
130 for appid in sortedids:
131 app = metadata.App(apps[appid])
135 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
137 for af in sorted(app.AntiFeatures):
138 wikidata += '{{AntiFeature|' + af + '}}\n'
143 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
146 app.added.strftime('%Y-%m-%d') if app.added else '',
147 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
163 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
165 wikidata += app.Summary
166 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
168 wikidata += "=Description=\n"
169 wikidata += metadata.description_wiki(app.Description) + "\n"
171 wikidata += "=Maintainer Notes=\n"
172 if app.MaintainerNotes:
173 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
174 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
176 # Get a list of all packages for this application...
178 gotcurrentver = False
182 if apk['packageName'] == appid:
183 if str(apk['versionCode']) == app.CurrentVersionCode:
186 # Include ones we can't build, as a special case...
187 for build in app.builds:
189 if build.versionCode == app.CurrentVersionCode:
191 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
192 apklist.append({'versionCode': int(build.versionCode),
193 'versionName': build.versionName,
194 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
199 if apk['versionCode'] == int(build.versionCode):
204 apklist.append({'versionCode': int(build.versionCode),
205 'versionName': build.versionName,
206 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
208 if app.CurrentVersionCode == '0':
210 # Sort with most recent first...
211 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
213 wikidata += "=Versions=\n"
214 if len(apklist) == 0:
215 wikidata += "We currently have no versions of this app available."
216 elif not gotcurrentver:
217 wikidata += "We don't have the current version of this app."
219 wikidata += "We have the current version of this app."
220 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
221 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
222 if len(app.NoSourceSince) > 0:
223 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
224 if len(app.CurrentVersion) > 0:
225 wikidata += "The current (recommended) version is " + app.CurrentVersion
226 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
229 wikidata += "==" + apk['versionName'] + "==\n"
231 if 'buildproblem' in apk:
232 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
235 wikidata += "This version is built and signed by "
237 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
239 wikidata += "the original developer.\n\n"
240 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
242 wikidata += '\n[[Category:' + wikicat + ']]\n'
243 if len(app.NoSourceSince) > 0:
244 wikidata += '\n[[Category:Apps missing source code]]\n'
245 if validapks == 0 and not app.Disabled:
246 wikidata += '\n[[Category:Apps with no packages]]\n'
247 if cantupdate and not app.Disabled:
248 wikidata += "\n[[Category:Apps we cannot update]]\n"
249 if buildfails and not app.Disabled:
250 wikidata += "\n[[Category:Apps with failing builds]]\n"
251 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
252 wikidata += '\n[[Category:Apps to Update]]\n'
254 wikidata += '\n[[Category:Apps that are disabled]]\n'
255 if app.UpdateCheckMode == 'None' and not app.Disabled:
256 wikidata += '\n[[Category:Apps with no update check]]\n'
257 for appcat in app.Categories:
258 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
260 # We can't have underscores in the page name, even if they're in
261 # the package ID, because MediaWiki messes with them...
262 pagename = appid.replace('_', ' ')
264 # Drop a trailing newline, because mediawiki is going to drop it anyway
265 # and it we don't we'll think the page has changed when it hasn't...
266 if wikidata.endswith('\n'):
267 wikidata = wikidata[:-1]
269 generated_pages[pagename] = wikidata
271 # Make a redirect from the name to the ID too, unless there's
272 # already an existing page with the name and it isn't a redirect.
274 apppagename = app.Name.replace('_', ' ')
275 apppagename = apppagename.replace('{', '')
276 apppagename = apppagename.replace('}', ' ')
277 apppagename = apppagename.replace(':', ' ')
278 apppagename = apppagename.replace('[', ' ')
279 apppagename = apppagename.replace(']', ' ')
280 # Drop double spaces caused mostly by replacing ':' above
281 apppagename = apppagename.replace(' ', ' ')
282 for expagename in site.allpages(prefix=apppagename,
283 filterredir='nonredirects',
285 if expagename == apppagename:
287 # Another reason not to make the redirect page is if the app name
288 # is the same as it's ID, because that will overwrite the real page
289 # with an redirect to itself! (Although it seems like an odd
290 # scenario this happens a lot, e.g. where there is metadata but no
291 # builds or binaries to extract a name from.
292 if apppagename == pagename:
295 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
297 for tcat, genp in [(wikicat, generated_pages),
298 (wikiredircat, generated_redirects)]:
299 catpages = site.Pages['Category:' + tcat]
301 for page in catpages:
302 existingpages.append(page.name)
303 if page.name in genp:
304 pagetxt = page.edit()
305 if pagetxt != genp[page.name]:
306 logging.debug("Updating modified page " + page.name)
307 page.save(genp[page.name], summary='Auto-updated')
309 logging.debug("Page " + page.name + " is unchanged")
311 logging.warn("Deleting page " + page.name)
312 page.delete('No longer published')
313 for pagename, text in genp.items():
314 logging.debug("Checking " + pagename)
315 if pagename not in existingpages:
316 logging.debug("Creating page " + pagename)
318 newpage = site.Pages[pagename]
319 newpage.save(text, summary='Auto-created')
320 except Exception as e:
321 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
323 # Purge server cache to ensure counts are up to date
324 site.pages['Repository Maintenance'].purge()
327 def delete_disabled_builds(apps, apkcache, repodirs):
328 """Delete disabled build outputs.
330 :param apps: list of all applications, as per metadata.read_metadata
331 :param apkcache: current apk cache information
332 :param repodirs: the repo directories to process
334 for appid, app in apps.items():
335 for build in app['builds']:
336 if not build.disable:
338 apkfilename = common.get_release_filename(app, build)
339 iconfilename = "%s.%s.png" % (
342 for repodir in repodirs:
344 os.path.join(repodir, apkfilename),
345 os.path.join(repodir, apkfilename + '.asc'),
346 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
348 for density in all_screen_densities:
349 repo_dir = get_icon_dir(repodir, density)
350 files.append(os.path.join(repo_dir, iconfilename))
353 if os.path.exists(f):
354 logging.info("Deleting disabled build output " + f)
356 if apkfilename in apkcache:
357 del apkcache[apkfilename]
360 def resize_icon(iconpath, density):
362 if not os.path.isfile(iconpath):
367 fp = open(iconpath, 'rb')
369 size = dpi_to_px(density)
371 if any(length > size for length in im.size):
373 im.thumbnail((size, size), Image.ANTIALIAS)
374 logging.debug("%s was too large at %s - new size is %s" % (
375 iconpath, oldsize, im.size))
376 im.save(iconpath, "PNG", optimize=True,
377 pnginfo=BLANK_PNG_INFO, icc_profile=None)
379 except Exception as e:
380 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
387 def resize_all_icons(repodirs):
388 """Resize all icons that exceed the max size
390 :param repodirs: the repo directories to process
392 for repodir in repodirs:
393 for density in screen_densities:
394 icon_dir = get_icon_dir(repodir, density)
395 icon_glob = os.path.join(icon_dir, '*.png')
396 for iconpath in glob.glob(icon_glob):
397 resize_icon(iconpath, density)
401 """ Get the signing certificate of an apk. To get the same md5 has that
402 Android gets, we encode the .RSA certificate in a specific format and pass
403 it hex-encoded to the md5 digest algorithm.
405 :param apkpath: path to the apk
406 :returns: A string containing the md5 of the signature of the apk or None
407 if an error occurred.
410 with zipfile.ZipFile(apkpath, 'r') as apk:
411 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
414 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
417 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
420 cert = apk.read(certs[0])
422 cert_encoded = common.get_certificate(cert)
424 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
427 def get_cache_file():
428 return os.path.join('tmp', 'apkcache')
432 """Get the cached dict of the APK index
434 Gather information about all the apk files in the repo directory,
435 using cached data if possible. Some of the index operations take a
436 long time, like calculating the SHA-256 and verifying the APK
439 The cache is invalidated if the metadata version is different, or
440 the 'allow_disabled_algorithms' config/option is different. In
441 those cases, there is no easy way to know what has changed from
442 the cache, so just rerun the whole thing.
447 apkcachefile = get_cache_file()
448 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
449 if not options.clean and os.path.exists(apkcachefile):
450 with open(apkcachefile, 'rb') as cf:
451 apkcache = pickle.load(cf, encoding='utf-8')
452 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
453 or apkcache.get('allow_disabled_algorithms') != ada:
458 apkcache["METADATA_VERSION"] = METADATA_VERSION
459 apkcache['allow_disabled_algorithms'] = ada
464 def write_cache(apkcache):
465 apkcachefile = get_cache_file()
466 cache_path = os.path.dirname(apkcachefile)
467 if not os.path.exists(cache_path):
468 os.makedirs(cache_path)
469 with open(apkcachefile, 'wb') as cf:
470 pickle.dump(apkcache, cf)
473 def get_icon_bytes(apkzip, iconsrc):
474 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
476 return apkzip.read(iconsrc)
478 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
481 def sha256sum(filename):
482 '''Calculate the sha256 of the given file'''
483 sha = hashlib.sha256()
484 with open(filename, 'rb') as f:
490 return sha.hexdigest()
493 def has_known_vulnerability(filename):
494 """checks for known vulnerabilities in the APK
496 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
497 version. Google also enforces this:
498 https://support.google.com/faqs/answer/6376725?hl=en
500 Checks whether there are more than one classes.dex or AndroidManifest.xml
501 files, which is invalid and an essential part of the "Master Key" attack.
502 http://www.saurik.com/id/17
504 Janus is similar to Master Key but is perhaps easier to scan for.
505 https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
510 # statically load this pattern
511 if not hasattr(has_known_vulnerability, "pattern"):
512 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
514 with open(filename.encode(), 'rb') as fp:
516 if first4 != b'\x50\x4b\x03\x04':
517 raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
518 .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
519 + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
522 with zipfile.ZipFile(filename) as zf:
523 for name in zf.namelist():
524 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
527 chunk = lib.read(4096)
530 m = has_known_vulnerability.pattern.search(chunk)
532 version = m.group(1).decode('ascii')
533 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
534 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
535 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
536 logging.debug(_('"{path}" contains recent {name} ({version})')
537 .format(path=filename, name=name, version=version))
539 logging.warning(_('"{path}" contains outdated {name} ({version})')
540 .format(path=filename, name=name, version=version))
543 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
544 if name in files_in_apk:
545 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
546 .format(apkfilename=filename, name=name))
548 files_in_apk.add(name)
552 def insert_obbs(repodir, apps, apks):
553 """Scans the .obb files in a given repo directory and adds them to the
554 relevant APK instances. OBB files have versionCodes like APK
555 files, and they are loosely associated. If there is an OBB file
556 present, then any APK with the same or higher versionCode will use
557 that OBB file. There are two OBB types: main and patch, each APK
558 can only have only have one of each.
560 https://developer.android.com/google/play/expansion-files.html
562 :param repodir: repo directory to scan
563 :param apps: list of current, valid apps
564 :param apks: current information on all APKs
568 def obbWarnDelete(f, msg):
569 logging.warning(msg + ' ' + f)
570 if options.delete_unknown:
571 logging.error(_("Deleting unknown file: {path}").format(path=f))
575 java_Integer_MIN_VALUE = -pow(2, 31)
576 currentPackageNames = apps.keys()
577 for f in glob.glob(os.path.join(repodir, '*.obb')):
578 obbfile = os.path.basename(f)
579 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
580 chunks = obbfile.split('.')
581 if chunks[0] != 'main' and chunks[0] != 'patch':
582 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
584 if not re.match(r'^-?[0-9]+$', chunks[1]):
585 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
586 .format(name=chunks[0]))
588 versionCode = int(chunks[1])
589 packagename = ".".join(chunks[2:-1])
591 highestVersionCode = java_Integer_MIN_VALUE
592 if packagename not in currentPackageNames:
593 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
596 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
597 highestVersionCode = apk['versionCode']
598 if versionCode > highestVersionCode:
599 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
600 .format(integer=str(versionCode)))
602 obbsha256 = sha256sum(f)
603 obbs.append((packagename, versionCode, obbfile, obbsha256))
606 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
607 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
608 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
609 apk['obbMainFile'] = obbfile
610 apk['obbMainFileSha256'] = obbsha256
611 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
612 apk['obbPatchFile'] = obbfile
613 apk['obbPatchFileSha256'] = obbsha256
614 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
618 def translate_per_build_anti_features(apps, apks):
619 """Grab the anti-features list from the build metadata
621 For most Anti-Features, they are really most applicable per-APK,
622 not for an app. An app can fix a vulnerability, add/remove
623 tracking, etc. This reads the 'antifeatures' list from the Build
624 entries in the fdroiddata metadata file, then transforms it into
625 the 'antiFeatures' list of unique items for the index.
627 The field key is all lower case in the metadata file to match the
628 rest of the Build fields. It is 'antiFeatures' camel case in the
629 implementation, index, and fdroidclient since it is translated
630 from the build 'antifeatures' field, not directly included.
634 antiFeatures = dict()
635 for packageName, app in apps.items():
637 for build in app['builds']:
638 afl = build.get('antifeatures')
640 d[int(build.versionCode)] = afl
642 antiFeatures[packageName] = d
645 d = antiFeatures.get(apk['packageName'])
647 afl = d.get(apk['versionCode'])
649 apk['antiFeatures'].update(afl)
652 def _get_localized_dict(app, locale):
653 '''get the dict to add localized store metadata to'''
654 if 'localized' not in app:
655 app['localized'] = collections.OrderedDict()
656 if locale not in app['localized']:
657 app['localized'][locale] = collections.OrderedDict()
658 return app['localized'][locale]
661 def _set_localized_text_entry(app, locale, key, f):
662 limit = config['char_limits'][key]
663 localized = _get_localized_dict(app, locale)
665 text = fp.read()[:limit]
667 localized[key] = text
670 def _set_author_entry(app, key, f):
671 limit = config['char_limits']['author']
673 text = fp.read()[:limit]
678 def _strip_and_copy_image(inpath, outpath):
679 """Remove any metadata from image and copy it to new path
681 Sadly, image metadata like EXIF can be used to exploit devices.
682 It is not used at all in the F-Droid ecosystem, so its much safer
683 just to remove it entirely.
687 extension = common.get_extension(inpath)[1]
688 if os.path.isdir(outpath):
689 outpath = os.path.join(outpath, os.path.basename(inpath))
690 if extension == 'png':
691 with open(inpath, 'rb') as fp:
692 in_image = Image.open(fp)
693 in_image.save(outpath, "PNG", optimize=True,
694 pnginfo=BLANK_PNG_INFO, icc_profile=None)
695 elif extension == 'jpg' or extension == 'jpeg':
696 with open(inpath, 'rb') as fp:
697 in_image = Image.open(fp)
698 data = list(in_image.getdata())
699 out_image = Image.new(in_image.mode, in_image.size)
700 out_image.putdata(data)
701 out_image.save(outpath, "JPEG", optimize=True)
703 raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
704 .format(extension=extension))
707 def copy_triple_t_store_metadata(apps):
708 """Include store metadata from the app's source repo
710 The Triple-T Gradle Play Publisher is a plugin that has a standard
711 file layout for all of the metadata and graphics that the Google
712 Play Store accepts. Since F-Droid has the git repo, it can just
713 pluck those files directly. This method reads any text files into
714 the app dict, then copies any graphics into the fdroid repo
717 This needs to be run before insert_localized_app_metadata() so that
718 the graphics files that are copied into the fdroid repo get
721 https://github.com/Triple-T/gradle-play-publisher#upload-images
722 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
726 if not os.path.isdir('build'):
727 return # nothing to do
729 for packageName, app in apps.items():
730 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
731 logging.debug('Triple-T Gradle Play Publisher: ' + d)
732 for root, dirs, files in os.walk(d):
733 segments = root.split('/')
734 locale = segments[-2]
736 if f == 'fulldescription':
737 _set_localized_text_entry(app, locale, 'description',
738 os.path.join(root, f))
740 elif f == 'shortdescription':
741 _set_localized_text_entry(app, locale, 'summary',
742 os.path.join(root, f))
745 _set_localized_text_entry(app, locale, 'name',
746 os.path.join(root, f))
749 _set_localized_text_entry(app, locale, 'video',
750 os.path.join(root, f))
752 elif f == 'whatsnew':
753 _set_localized_text_entry(app, segments[-1], 'whatsNew',
754 os.path.join(root, f))
756 elif f == 'contactEmail':
757 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
759 elif f == 'contactPhone':
760 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
762 elif f == 'contactWebsite':
763 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
766 base, extension = common.get_extension(f)
767 dirname = os.path.basename(root)
768 if extension in ALLOWED_EXTENSIONS \
769 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
770 if segments[-2] == 'listing':
771 locale = segments[-3]
773 locale = segments[-2]
774 destdir = os.path.join('repo', packageName, locale, dirname)
775 os.makedirs(destdir, mode=0o755, exist_ok=True)
776 sourcefile = os.path.join(root, f)
777 destfile = os.path.join(destdir, os.path.basename(f))
778 logging.debug('copying ' + sourcefile + ' ' + destfile)
779 _strip_and_copy_image(sourcefile, destfile)
782 def insert_localized_app_metadata(apps):
783 """scans standard locations for graphics and localized text
785 Scans for localized description files, store graphics, and
786 screenshot PNG files in statically defined screenshots directory
787 and adds them to the app metadata. The screenshots and graphic
788 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
789 and must be in the following layout:
790 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
792 repo/packageName/locale/featureGraphic.png
793 repo/packageName/locale/phoneScreenshots/1.png
794 repo/packageName/locale/phoneScreenshots/2.png
796 The changelog files must be text files named with the versionCode
797 ending with ".txt" and must be in the following layout:
798 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
800 repo/packageName/locale/changelogs/12345.txt
802 This will scan the each app's source repo then the metadata/ dir
803 for these standard locations of changelog files. If it finds
804 them, they will be added to the dict of all packages, with the
805 versions in the metadata/ folder taking precendence over the what
806 is in the app's source repo.
808 Where "packageName" is the app's packageName and "locale" is the locale
809 of the graphics, e.g. what language they are in, using the IETF RFC5646
810 format (en-US, fr-CA, es-MX, etc).
812 This will also scan the app's git for a fastlane folder, and the
813 metadata/ folder and the apps' source repos for standard locations
814 of graphic and screenshot files. If it finds them, it will copy
815 them into the repo. The fastlane files follow this pattern:
816 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
820 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
821 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
822 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
823 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
825 for srcd in sorted(sourcedirs):
826 if not os.path.isdir(srcd):
828 for root, dirs, files in os.walk(srcd):
829 segments = root.split('/')
830 packageName = segments[1]
831 if packageName not in apps:
832 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
834 locale = segments[-1]
835 destdir = os.path.join('repo', packageName, locale)
837 # flavours specified in build receipt
839 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
840 and 'gradle' in apps[packageName].builds[-1]:
841 build_flavours = apps[packageName].builds[-1].gradle
843 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
844 logging.debug("ignoring due to wrong flavour")
848 if f in ('description.txt', 'full_description.txt'):
849 _set_localized_text_entry(apps[packageName], locale, 'description',
850 os.path.join(root, f))
852 elif f in ('summary.txt', 'short_description.txt'):
853 _set_localized_text_entry(apps[packageName], locale, 'summary',
854 os.path.join(root, f))
856 elif f in ('name.txt', 'title.txt'):
857 _set_localized_text_entry(apps[packageName], locale, 'name',
858 os.path.join(root, f))
860 elif f == 'video.txt':
861 _set_localized_text_entry(apps[packageName], locale, 'video',
862 os.path.join(root, f))
864 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
865 locale = segments[-2]
866 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
867 os.path.join(root, f))
870 base, extension = common.get_extension(f)
871 if locale == 'images':
872 locale = segments[-2]
873 destdir = os.path.join('repo', packageName, locale)
874 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
875 os.makedirs(destdir, mode=0o755, exist_ok=True)
876 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
877 _strip_and_copy_image(os.path.join(root, f), destdir)
879 if d in SCREENSHOT_DIRS:
880 if locale == 'images':
881 locale = segments[-2]
882 destdir = os.path.join('repo', packageName, locale)
883 for f in glob.glob(os.path.join(root, d, '*.*')):
884 _ignored, extension = common.get_extension(f)
885 if extension in ALLOWED_EXTENSIONS:
886 screenshotdestdir = os.path.join(destdir, d)
887 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
888 logging.debug('copying ' + f + ' ' + screenshotdestdir)
889 _strip_and_copy_image(f, screenshotdestdir)
891 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
893 if not os.path.isdir(d):
895 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
896 if not os.path.isfile(f):
898 segments = f.split('/')
899 packageName = segments[1]
901 screenshotdir = segments[3]
902 filename = os.path.basename(f)
903 base, extension = common.get_extension(filename)
905 if packageName not in apps:
906 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
907 .format(path=filename, name=packageName))
909 graphics = _get_localized_dict(apps[packageName], locale)
911 if extension not in ALLOWED_EXTENSIONS:
912 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
913 elif base in GRAPHIC_NAMES:
914 # there can only be zero or one of these per locale
915 graphics[base] = filename
916 elif screenshotdir in SCREENSHOT_DIRS:
917 # there can any number of these per locale
918 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
919 if screenshotdir not in graphics:
920 graphics[screenshotdir] = []
921 graphics[screenshotdir].append(filename)
923 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
926 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
927 """Scan a repo for all files with an extension except APK/OBB
929 :param apkcache: current cached info about all repo files
930 :param repodir: repo directory to scan
931 :param knownapks: list of all known files, as per metadata.read_metadata
932 :param use_date_from_file: use date from file (instead of current date)
933 for newly added files
938 repodir = repodir.encode('utf-8')
939 for name in os.listdir(repodir):
940 file_extension = common.get_file_extension(name)
941 if file_extension == 'apk' or file_extension == 'obb':
943 filename = os.path.join(repodir, name)
944 name_utf8 = name.decode('utf-8')
945 if filename.endswith(b'_src.tar.gz'):
946 logging.debug(_('skipping source tarball: {path}')
947 .format(path=filename.decode('utf-8')))
949 if not common.is_repo_file(filename):
951 stat = os.stat(filename)
952 if stat.st_size == 0:
953 raise FDroidException(_('{path} is zero size!')
954 .format(path=filename))
956 shasum = sha256sum(filename)
959 repo_file = apkcache[name]
960 # added time is cached as tuple but used here as datetime instance
961 if 'added' in repo_file:
962 a = repo_file['added']
963 if isinstance(a, datetime):
964 repo_file['added'] = a
966 repo_file['added'] = datetime(*a[:6])
967 if repo_file.get('hash') == shasum:
968 logging.debug(_("Reading {apkfilename} from cache")
969 .format(apkfilename=name_utf8))
972 logging.debug(_("Ignoring stale cache data for {apkfilename}")
973 .format(apkfilename=name_utf8))
976 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
977 repo_file = collections.OrderedDict()
978 repo_file['name'] = os.path.splitext(name_utf8)[0]
979 # TODO rename apkname globally to something more generic
980 repo_file['apkName'] = name_utf8
981 repo_file['hash'] = shasum
982 repo_file['hashType'] = 'sha256'
983 repo_file['versionCode'] = 0
984 repo_file['versionName'] = shasum
985 # the static ID is the SHA256 unless it is set in the metadata
986 repo_file['packageName'] = shasum
988 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
990 repo_file['packageName'] = m.group(1)
991 repo_file['versionCode'] = int(m.group(2))
992 srcfilename = name + b'_src.tar.gz'
993 if os.path.exists(os.path.join(repodir, srcfilename)):
994 repo_file['srcname'] = srcfilename.decode('utf-8')
995 repo_file['size'] = stat.st_size
997 apkcache[name] = repo_file
1000 if use_date_from_file:
1001 timestamp = stat.st_ctime
1002 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1004 default_date_param = None
1006 # Record in knownapks, getting the added date at the same time..
1007 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1008 default_date=default_date_param)
1010 repo_file['added'] = added
1012 repo_files.append(repo_file)
1014 return repo_files, cachechanged
1017 def scan_apk(apk_file):
1019 Scans an APK file and returns dictionary with metadata of the APK.
1021 Attention: This does *not* verify that the APK signature is correct.
1023 :param apk_file: The (ideally absolute) path to the APK file
1024 :raises BuildException
1025 :return A dict containing APK metadata
1028 'hash': sha256sum(apk_file),
1029 'hashType': 'sha256',
1030 'uses-permission': [],
1031 'uses-permission-sdk-23': [],
1035 'antiFeatures': set(),
1038 if SdkToolsPopen(['aapt', 'version'], output=False):
1039 scan_apk_aapt(apk, apk_file)
1041 scan_apk_androguard(apk, apk_file)
1043 # Get the signature, or rather the signing key fingerprints
1044 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1045 apk['sig'] = getsig(apk_file)
1047 raise BuildException("Failed to get apk signature")
1048 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1050 if not apk.get('signer'):
1051 raise BuildException("Failed to get apk signing key fingerprint")
1053 # Get size of the APK
1054 apk['size'] = os.path.getsize(apk_file)
1056 if 'minSdkVersion' not in apk:
1057 logging.warning("No SDK version information found in {0}".format(apk_file))
1058 apk['minSdkVersion'] = 1
1060 # Check for known vulnerabilities
1061 if has_known_vulnerability(apk_file):
1062 apk['antiFeatures'].add('KnownVuln')
1067 def scan_apk_aapt(apk, apkfile):
1068 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1069 if p.returncode != 0:
1070 if options.delete_unknown:
1071 if os.path.exists(apkfile):
1072 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1075 logging.error("Could not find {0} to remove it".format(apkfile))
1077 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1078 raise BuildException(_("Invalid APK"))
1079 for line in p.output.splitlines():
1080 if line.startswith("package:"):
1082 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1083 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1084 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1085 except Exception as e:
1086 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1087 elif line.startswith("application:"):
1088 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1089 # Keep path to non-dpi icon in case we need it
1090 match = re.match(APK_ICON_PAT_NODPI, line)
1092 apk['icons_src']['-1'] = match.group(1)
1093 elif line.startswith("launchable-activity:"):
1094 # Only use launchable-activity as fallback to application
1096 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1097 if '-1' not in apk['icons_src']:
1098 match = re.match(APK_ICON_PAT_NODPI, line)
1100 apk['icons_src']['-1'] = match.group(1)
1101 elif line.startswith("application-icon-"):
1102 match = re.match(APK_ICON_PAT, line)
1104 density = match.group(1)
1105 path = match.group(2)
1106 apk['icons_src'][density] = path
1107 elif line.startswith("sdkVersion:"):
1108 m = re.match(APK_SDK_VERSION_PAT, line)
1110 logging.error(line.replace('sdkVersion:', '')
1111 + ' is not a valid minSdkVersion!')
1113 apk['minSdkVersion'] = m.group(1)
1114 # if target not set, default to min
1115 if 'targetSdkVersion' not in apk:
1116 apk['targetSdkVersion'] = m.group(1)
1117 elif line.startswith("targetSdkVersion:"):
1118 m = re.match(APK_SDK_VERSION_PAT, line)
1120 logging.error(line.replace('targetSdkVersion:', '')
1121 + ' is not a valid targetSdkVersion!')
1123 apk['targetSdkVersion'] = m.group(1)
1124 elif line.startswith("maxSdkVersion:"):
1125 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1126 elif line.startswith("native-code:"):
1127 apk['nativecode'] = []
1128 for arch in line[13:].split(' '):
1129 apk['nativecode'].append(arch[1:-1])
1130 elif line.startswith('uses-permission:'):
1131 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1132 if perm_match['maxSdkVersion']:
1133 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1134 permission = UsesPermission(
1136 perm_match['maxSdkVersion']
1139 apk['uses-permission'].append(permission)
1140 elif line.startswith('uses-permission-sdk-23:'):
1141 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1142 if perm_match['maxSdkVersion']:
1143 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1144 permission_sdk_23 = UsesPermissionSdk23(
1146 perm_match['maxSdkVersion']
1149 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1151 elif line.startswith('uses-feature:'):
1152 feature = re.match(APK_FEATURE_PAT, line).group(1)
1153 # Filter out this, it's only added with the latest SDK tools and
1154 # causes problems for lots of apps.
1155 if feature != "android.hardware.screen.portrait" \
1156 and feature != "android.hardware.screen.landscape":
1157 if feature.startswith("android.feature."):
1158 feature = feature[16:]
1159 apk['features'].add(feature)
1162 def scan_apk_androguard(apk, apkfile):
1164 from androguard.core.bytecodes.apk import APK
1165 apkobject = APK(apkfile)
1166 if apkobject.is_valid_APK():
1167 arsc = apkobject.get_android_resources()
1169 if options.delete_unknown:
1170 if os.path.exists(apkfile):
1171 logging.error(_("Failed to get apk information, deleting {path}")
1172 .format(path=apkfile))
1175 logging.error(_("Could not find {path} to remove it")
1176 .format(path=apkfile))
1178 logging.error(_("Failed to get apk information, skipping {path}")
1179 .format(path=apkfile))
1180 raise BuildException(_("Invalid APK"))
1182 raise FDroidException("androguard library is not installed and aapt not present")
1183 except FileNotFoundError:
1184 logging.error(_("Could not open apk file for analysis"))
1185 raise BuildException(_("Invalid APK"))
1187 apk['packageName'] = apkobject.get_package()
1188 apk['versionCode'] = int(apkobject.get_androidversion_code())
1189 apk['versionName'] = apkobject.get_androidversion_name()
1190 if apk['versionName'][0] == "@":
1191 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1192 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1193 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1194 apk['name'] = apkobject.get_app_name()
1196 if apkobject.get_max_sdk_version() is not None:
1197 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1198 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1199 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1201 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1202 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1204 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1206 for file in apkobject.get_files():
1207 d_re = density_re.match(file)
1209 folder = d_re.group(1).split('-')
1211 resolution = folder[1]
1214 density = screen_resolutions[resolution]
1215 apk['icons_src'][density] = d_re.group(0)
1217 if apk['icons_src'].get('-1') is None:
1218 apk['icons_src']['-1'] = apk['icons_src']['160']
1220 arch_re = re.compile("^lib/(.*)/.*$")
1221 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1223 apk['nativecode'] = []
1224 apk['nativecode'].extend(sorted(list(arch)))
1226 xml = apkobject.get_android_manifest_xml()
1228 for item in xml.getElementsByTagName('uses-permission'):
1229 name = str(item.getAttribute("android:name"))
1230 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1231 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1232 permission = UsesPermission(
1236 apk['uses-permission'].append(permission)
1238 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1239 name = str(item.getAttribute("android:name"))
1240 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1241 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1242 permission_sdk_23 = UsesPermissionSdk23(
1246 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1248 for item in xml.getElementsByTagName('uses-feature'):
1249 feature = str(item.getAttribute("android:name"))
1250 if feature != "android.hardware.screen.portrait" \
1251 and feature != "android.hardware.screen.landscape":
1252 if feature.startswith("android.feature."):
1253 feature = feature[16:]
1254 apk['features'].append(feature)
1257 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1258 allow_disabled_algorithms=False, archive_bad_sig=False):
1259 """Processes the apk with the given filename in the given repo directory.
1261 This also extracts the icons.
1263 :param apkcache: current apk cache information
1264 :param apkfilename: the filename of the apk to scan
1265 :param repodir: repo directory to scan
1266 :param knownapks: known apks info
1267 :param use_date_from_apk: use date from APK (instead of current date)
1268 for newly added APKs
1269 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1270 disabled algorithms in the signature (e.g. MD5)
1271 :param archive_bad_sig: move APKs with a bad signature to the archive
1272 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1273 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1277 apkfile = os.path.join(repodir, apkfilename)
1279 cachechanged = False
1281 if apkfilename in apkcache:
1282 apk = apkcache[apkfilename]
1283 if apk.get('hash') == sha256sum(apkfile):
1284 logging.debug(_("Reading {apkfilename} from cache")
1285 .format(apkfilename=apkfilename))
1288 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1289 .format(apkfilename=apkfilename))
1292 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1295 apk = scan_apk(apkfile)
1296 except BuildException:
1297 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1298 .format(apkfilename=apkfilename))
1299 return True, None, False
1301 # Check for debuggable apks...
1302 if common.isApkAndDebuggable(apkfile):
1303 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1305 if options.rename_apks:
1306 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1307 std_short_name = os.path.join(repodir, n)
1308 if apkfile != std_short_name:
1309 if os.path.exists(std_short_name):
1310 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1311 if apkfile != std_long_name:
1312 if os.path.exists(std_long_name):
1313 dupdir = os.path.join('duplicates', repodir)
1314 if not os.path.isdir(dupdir):
1315 os.makedirs(dupdir, exist_ok=True)
1316 dupfile = os.path.join('duplicates', std_long_name)
1317 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1318 os.rename(apkfile, dupfile)
1319 return True, None, False
1321 os.rename(apkfile, std_long_name)
1322 apkfile = std_long_name
1324 os.rename(apkfile, std_short_name)
1325 apkfile = std_short_name
1326 apkfilename = apkfile[len(repodir) + 1:]
1328 apk['apkName'] = apkfilename
1329 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1330 if os.path.exists(os.path.join(repodir, srcfilename)):
1331 apk['srcname'] = srcfilename
1333 # verify the jar signature is correct, allow deprecated
1334 # algorithms only if the APK is in the archive.
1336 if not common.verify_apk_signature(apkfile):
1337 if repodir == 'archive' or allow_disabled_algorithms:
1338 if common.verify_old_apk_signature(apkfile):
1339 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1347 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1348 .format(apkfilename=apkfilename))
1349 move_apk_between_sections(repodir, 'archive', apk)
1351 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1352 .format(apkfilename=apkfilename))
1353 return True, None, False
1355 apkzip = zipfile.ZipFile(apkfile, 'r')
1357 manifest = apkzip.getinfo('AndroidManifest.xml')
1358 # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1359 if (1980, 0, 0) != manifest.date_time[0:3]:
1361 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1362 except ValueError as e:
1363 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1364 .format(apkfilename=apkfile) + str(e))
1366 # extract icons from APK zip file
1367 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1369 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1371 apkzip.close() # ensure that APK zip file gets closed
1373 # resize existing icons for densities missing in the APK
1374 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1376 if use_date_from_apk and manifest.date_time[1] != 0:
1377 default_date_param = datetime(*manifest.date_time)
1379 default_date_param = None
1381 # Record in known apks, getting the added date at the same time..
1382 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1383 default_date=default_date_param)
1385 apk['added'] = added
1387 apkcache[apkfilename] = apk
1390 return False, apk, cachechanged
1393 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1394 """Processes the apks in the given repo directory.
1396 This also extracts the icons.
1398 :param apkcache: current apk cache information
1399 :param repodir: repo directory to scan
1400 :param knownapks: known apks info
1401 :param use_date_from_apk: use date from APK (instead of current date)
1402 for newly added APKs
1403 :returns: (apks, cachechanged) where apks is a list of apk information,
1404 and cachechanged is True if the apkcache got changed.
1407 cachechanged = False
1409 for icon_dir in get_all_icon_dirs(repodir):
1410 if os.path.exists(icon_dir):
1412 shutil.rmtree(icon_dir)
1413 os.makedirs(icon_dir)
1415 os.makedirs(icon_dir)
1418 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1419 apkfilename = apkfile[len(repodir) + 1:]
1420 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1421 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1422 use_date_from_apk, ada, True)
1426 cachechanged = cachechanged or cachethis
1428 return apks, cachechanged
1431 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1433 Extracts icons from the given APK zip in various densities,
1434 saves them into given repo directory
1435 and stores their names in the APK metadata dictionary.
1437 :param icon_filename: A string representing the icon's file name
1438 :param apk: A populated dictionary containing APK metadata.
1439 Needs to have 'icons_src' key
1440 :param apkzip: An opened zipfile.ZipFile of the APK file
1441 :param repo_dir: The directory of the APK's repository
1442 :return: A list of icon densities that are missing
1444 empty_densities = []
1445 for density in screen_densities:
1446 if density not in apk['icons_src']:
1447 empty_densities.append(density)
1449 icon_src = apk['icons_src'][density]
1450 icon_dir = get_icon_dir(repo_dir, density)
1451 icon_dest = os.path.join(icon_dir, icon_filename)
1453 # Extract the icon files per density
1454 if icon_src.endswith('.xml'):
1455 png = os.path.basename(icon_src)[:-4] + '.png'
1456 for f in apkzip.namelist():
1458 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1459 if m and screen_resolutions[m.group(2)] == density:
1461 if icon_src.endswith('.xml'):
1462 empty_densities.append(density)
1465 with open(icon_dest, 'wb') as f:
1466 f.write(get_icon_bytes(apkzip, icon_src))
1467 apk['icons'][density] = icon_filename
1468 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1469 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1470 del apk['icons_src'][density]
1471 empty_densities.append(density)
1473 if '-1' in apk['icons_src']:
1474 icon_src = apk['icons_src']['-1']
1475 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1476 with open(icon_path, 'wb') as f:
1477 f.write(get_icon_bytes(apkzip, icon_src))
1479 im = Image.open(icon_path)
1480 dpi = px_to_dpi(im.size[0])
1481 for density in screen_densities:
1482 if density in apk['icons']:
1484 if density == screen_densities[-1] or dpi >= int(density):
1485 apk['icons'][density] = icon_filename
1486 shutil.move(icon_path,
1487 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1488 empty_densities.remove(density)
1490 except Exception as e:
1491 logging.warning(_("Failed reading {path}: {error}")
1492 .format(path=icon_path, error=e))
1497 apk['icon'] = icon_filename
1499 return empty_densities
1502 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1504 Resize existing icons for densities missing in the APK to ensure all densities are available
1506 :param empty_densities: A list of icon densities that are missing
1507 :param icon_filename: A string representing the icon's file name
1508 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1509 :param repo_dir: The directory of the APK's repository
1511 # First try resizing down to not lose quality
1513 for density in screen_densities:
1514 if density not in empty_densities:
1515 last_density = density
1517 if last_density is None:
1519 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1521 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1522 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1525 fp = open(last_icon_path, 'rb')
1528 size = dpi_to_px(density)
1530 im.thumbnail((size, size), Image.ANTIALIAS)
1531 im.save(icon_path, "PNG", optimize=True,
1532 pnginfo=BLANK_PNG_INFO, icc_profile=None)
1533 empty_densities.remove(density)
1534 except Exception as e:
1535 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1540 # Then just copy from the highest resolution available
1542 for density in reversed(screen_densities):
1543 if density not in empty_densities:
1544 last_density = density
1547 if last_density is None:
1551 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1552 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1554 empty_densities.remove(density)
1556 for density in screen_densities:
1557 icon_dir = get_icon_dir(repo_dir, density)
1558 icon_dest = os.path.join(icon_dir, icon_filename)
1559 resize_icon(icon_dest, density)
1561 # Copy from icons-mdpi to icons since mdpi is the baseline density
1562 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1563 if os.path.isfile(baseline):
1564 apk['icons']['0'] = icon_filename
1565 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1568 def apply_info_from_latest_apk(apps, apks):
1570 Some information from the apks needs to be applied up to the application level.
1571 When doing this, we use the info from the most recent version's apk.
1572 We deal with figuring out when the app was added and last updated at the same time.
1574 for appid, app in apps.items():
1575 bestver = UNSET_VERSION_CODE
1577 if apk['packageName'] == appid:
1578 if apk['versionCode'] > bestver:
1579 bestver = apk['versionCode']
1583 if not app.added or apk['added'] < app.added:
1584 app.added = apk['added']
1585 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1586 app.lastUpdated = apk['added']
1589 logging.debug("Don't know when " + appid + " was added")
1590 if not app.lastUpdated:
1591 logging.debug("Don't know when " + appid + " was last updated")
1593 if bestver == UNSET_VERSION_CODE:
1595 if app.Name is None:
1596 app.Name = app.AutoName or appid
1598 logging.debug("Application " + appid + " has no packages")
1600 if app.Name is None:
1601 app.Name = bestapk['name']
1602 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1603 if app.CurrentVersionCode is None:
1604 app.CurrentVersionCode = str(bestver)
1607 def make_categories_txt(repodir, categories):
1608 '''Write a category list in the repo to allow quick access'''
1610 for cat in sorted(categories):
1611 catdata += cat + '\n'
1612 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1616 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1618 def filter_apk_list_sorted(apk_list):
1620 for apk in apk_list:
1621 if apk['packageName'] == appid:
1624 # Sort the apk list by version code. First is highest/newest.
1625 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1627 for appid, app in apps.items():
1629 if app.ArchivePolicy:
1630 keepversions = int(app.ArchivePolicy[:-9])
1632 keepversions = defaultkeepversions
1634 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1635 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1637 current_app_apks = filter_apk_list_sorted(apks)
1638 if len(current_app_apks) > keepversions:
1639 # Move back the ones we don't want.
1640 for apk in current_app_apks[keepversions:]:
1641 move_apk_between_sections(repodir, archivedir, apk)
1642 archapks.append(apk)
1645 current_app_archapks = filter_apk_list_sorted(archapks)
1646 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1648 # Move forward the ones we want again, except DisableAlgorithm
1649 for apk in current_app_archapks:
1650 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1651 move_apk_between_sections(archivedir, repodir, apk)
1652 archapks.remove(apk)
1655 if kept == keepversions:
1659 def move_apk_between_sections(from_dir, to_dir, apk):
1660 """move an APK from repo to archive or vice versa"""
1662 def _move_file(from_dir, to_dir, filename, ignore_missing):
1663 from_path = os.path.join(from_dir, filename)
1664 if ignore_missing and not os.path.exists(from_path):
1666 to_path = os.path.join(to_dir, filename)
1667 if not os.path.exists(to_dir):
1669 shutil.move(from_path, to_path)
1671 if from_dir == to_dir:
1674 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1675 _move_file(from_dir, to_dir, apk['apkName'], False)
1676 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1677 for density in all_screen_densities:
1678 from_icon_dir = get_icon_dir(from_dir, density)
1679 to_icon_dir = get_icon_dir(to_dir, density)
1680 if density not in apk.get('icons', []):
1682 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1683 if 'srcname' in apk:
1684 _move_file(from_dir, to_dir, apk['srcname'], False)
1687 def add_apks_to_per_app_repos(repodir, apks):
1688 apks_per_app = dict()
1690 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1691 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1692 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1693 apks_per_app[apk['packageName']] = apk
1695 if not os.path.exists(apk['per_app_icons']):
1696 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1697 os.makedirs(apk['per_app_icons'])
1699 apkpath = os.path.join(repodir, apk['apkName'])
1700 shutil.copy(apkpath, apk['per_app_repo'])
1701 apksigpath = apkpath + '.sig'
1702 if os.path.exists(apksigpath):
1703 shutil.copy(apksigpath, apk['per_app_repo'])
1704 apkascpath = apkpath + '.asc'
1705 if os.path.exists(apkascpath):
1706 shutil.copy(apkascpath, apk['per_app_repo'])
1709 def create_metadata_from_template(apk):
1710 '''create a new metadata file using internal or external template
1712 Generate warnings for apk's with no metadata (or create skeleton
1713 metadata files, if requested on the command line). Though the
1714 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1715 since those impose things on the metadata file made from the
1716 template: field sort order, empty field value, formatting, etc.
1720 if os.path.exists('template.yml'):
1721 with open('template.yml') as f:
1723 if 'name' in apk and apk['name'] != '':
1724 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1725 r'\1 ' + apk['name'],
1727 flags=re.IGNORECASE | re.MULTILINE)
1729 logging.warning(_('{appid} does not have a name! Using package name instead.')
1730 .format(appid=apk['packageName']))
1731 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1732 r'\1 ' + apk['packageName'],
1734 flags=re.IGNORECASE | re.MULTILINE)
1735 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1739 app['Categories'] = [os.path.basename(os.getcwd())]
1740 # include some blanks as part of the template
1741 app['AuthorName'] = ''
1744 app['IssueTracker'] = ''
1745 app['SourceCode'] = ''
1746 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1747 if 'name' in apk and apk['name'] != '':
1748 app['Name'] = apk['name']
1750 logging.warning(_('{appid} does not have a name! Using package name instead.')
1751 .format(appid=apk['packageName']))
1752 app['Name'] = apk['packageName']
1753 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1754 yaml.dump(app, f, default_flow_style=False)
1755 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1764 global config, options
1766 # Parse command line...
1767 parser = ArgumentParser()
1768 common.setup_global_opts(parser)
1769 parser.add_argument("--create-key", action="store_true", default=False,
1770 help=_("Add a repo signing key to an unsigned repo"))
1771 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1772 help=_("Add skeleton metadata files for APKs that are missing them"))
1773 parser.add_argument("--delete-unknown", action="store_true", default=False,
1774 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1775 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1776 help=_("Report on build data status"))
1777 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1778 help=_("Interactively ask about things that need updating."))
1779 parser.add_argument("-I", "--icons", action="store_true", default=False,
1780 help=_("Resize all the icons exceeding the max pixel size and exit"))
1781 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1782 help=_("Specify editor to use in interactive mode. Default " +
1783 "is {path}").format(path='/etc/alternatives/editor'))
1784 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1785 help=_("Update the wiki"))
1786 parser.add_argument("--pretty", action="store_true", default=False,
1787 help=_("Produce human-readable XML/JSON for index files"))
1788 parser.add_argument("--clean", action="store_true", default=False,
1789 help=_("Clean update - don't uses caches, reprocess all APKs"))
1790 parser.add_argument("--nosign", action="store_true", default=False,
1791 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1792 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1793 help=_("Use date from APK instead of current time for newly added APKs"))
1794 parser.add_argument("--rename-apks", action="store_true", default=False,
1795 help=_("Rename APK files that do not match package.name_123.apk"))
1796 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1797 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1798 metadata.add_metadata_arguments(parser)
1799 options = parser.parse_args()
1800 metadata.warnings_action = options.W
1802 config = common.read_config(options)
1804 if not ('jarsigner' in config and 'keytool' in config):
1805 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1808 if config['archive_older'] != 0:
1809 repodirs.append('archive')
1810 if not os.path.exists('archive'):
1814 resize_all_icons(repodirs)
1817 if options.rename_apks:
1818 options.clean = True
1820 # check that icons exist now, rather than fail at the end of `fdroid update`
1821 for k in ['repo_icon', 'archive_icon']:
1823 if not os.path.exists(config[k]):
1824 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1825 .format(name=k, path=config[k]))
1828 # if the user asks to create a keystore, do it now, reusing whatever it can
1829 if options.create_key:
1830 if os.path.exists(config['keystore']):
1831 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1832 logging.critical("\t'" + config['keystore'] + "'")
1835 if 'repo_keyalias' not in config:
1836 config['repo_keyalias'] = socket.getfqdn()
1837 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1838 if 'keydname' not in config:
1839 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1840 common.write_to_config(config, 'keydname', config['keydname'])
1841 if 'keystore' not in config:
1842 config['keystore'] = common.default_config['keystore']
1843 common.write_to_config(config, 'keystore', config['keystore'])
1845 password = common.genpassword()
1846 if 'keystorepass' not in config:
1847 config['keystorepass'] = password
1848 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1849 if 'keypass' not in config:
1850 config['keypass'] = password
1851 common.write_to_config(config, 'keypass', config['keypass'])
1852 common.genkeystore(config)
1855 apps = metadata.read_metadata()
1857 # Generate a list of categories...
1859 for app in apps.values():
1860 categories.update(app.Categories)
1862 # Read known apks data (will be updated and written back when we've finished)
1863 knownapks = common.KnownApks()
1866 apkcache = get_cache()
1868 # Delete builds for disabled apps
1869 delete_disabled_builds(apps, apkcache, repodirs)
1871 # Scan all apks in the main repo
1872 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1874 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1875 options.use_date_from_apk)
1876 cachechanged = cachechanged or fcachechanged
1879 if apk['packageName'] not in apps:
1880 if options.create_metadata:
1881 create_metadata_from_template(apk)
1882 apps = metadata.read_metadata()
1884 msg = _("{apkfilename} ({appid}) has no metadata!") \
1885 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1886 if options.delete_unknown:
1887 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1888 .format(apkfilename=apk['apkName']))
1889 rmf = os.path.join(repodirs[0], apk['apkName'])
1890 if not os.path.exists(rmf):
1891 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1895 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1897 copy_triple_t_store_metadata(apps)
1898 insert_obbs(repodirs[0], apps, apks)
1899 insert_localized_app_metadata(apps)
1900 translate_per_build_anti_features(apps, apks)
1902 # Scan the archive repo for apks as well
1903 if len(repodirs) > 1:
1904 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1910 # Apply information from latest apks to the application and update dates
1911 apply_info_from_latest_apk(apps, apks + archapks)
1913 # Sort the app list by name, then the web site doesn't have to by default.
1914 # (we had to wait until we'd scanned the apks to do this, because mostly the
1915 # name comes from there!)
1916 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1918 # APKs are placed into multiple repos based on the app package, providing
1919 # per-app subscription feeds for nightly builds and things like it
1920 if config['per_app_repos']:
1921 add_apks_to_per_app_repos(repodirs[0], apks)
1922 for appid, app in apps.items():
1923 repodir = os.path.join(appid, 'fdroid', 'repo')
1925 appdict[appid] = app
1926 if os.path.isdir(repodir):
1927 index.make(appdict, [appid], apks, repodir, False)
1929 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1932 if len(repodirs) > 1:
1933 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1935 # Make the index for the main repo...
1936 index.make(apps, sortedids, apks, repodirs[0], False)
1937 make_categories_txt(repodirs[0], categories)
1939 # If there's an archive repo, make the index for it. We already scanned it
1941 if len(repodirs) > 1:
1942 index.make(apps, sortedids, archapks, repodirs[1], True)
1944 git_remote = config.get('binary_transparency_remote')
1945 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1947 btlog.make_binary_transparency_log(repodirs)
1949 if config['update_stats']:
1950 # Update known apks info...
1951 knownapks.writeifchanged()
1953 # Generate latest apps data for widget
1954 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1956 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1958 appid = line.rstrip()
1959 data += appid + "\t"
1961 data += app.Name + "\t"
1962 if app.icon is not None:
1963 data += app.icon + "\t"
1964 data += app.License + "\n"
1965 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1969 write_cache(apkcache)
1971 # Update the wiki...
1973 update_wiki(apps, sortedids, apks + archapks)
1975 logging.info(_("Finished"))
1978 if __name__ == "__main__":