3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
38 from PIL import Image, PngImagePlugin
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
90 def dpi_to_px(density):
91 return (int(density) * 48) / 160
95 return (int(px) * 160) / 48
98 def get_icon_dir(repodir, density):
100 return os.path.join(repodir, "icons")
101 return os.path.join(repodir, "icons-%s" % density)
104 def get_icon_dirs(repodir):
105 for density in screen_densities:
106 yield get_icon_dir(repodir, density)
109 def get_all_icon_dirs(repodir):
110 for density in all_screen_densities:
111 yield get_icon_dir(repodir, density)
114 def update_wiki(apps, sortedids, apks):
117 :param apps: fully populated list of all applications
118 :param apks: all apks, except...
120 logging.info("Updating wiki")
122 wikiredircat = 'App Redirects'
124 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
125 path=config['wiki_path'])
126 site.login(config['wiki_user'], config['wiki_password'])
128 generated_redirects = {}
130 for appid in sortedids:
131 app = metadata.App(apps[appid])
135 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
137 for af in sorted(app.AntiFeatures):
138 wikidata += '{{AntiFeature|' + af + '}}\n'
143 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s|activity=%s}}\n' % (
146 app.added.strftime('%Y-%m-%d') if app.added else '',
147 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
161 'https://gitlab.com/search?group_id=28397&scope=issues&search=' + appid,
165 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
167 wikidata += app.Summary
168 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
170 wikidata += "=Description=\n"
171 wikidata += metadata.description_wiki(app.Description) + "\n"
173 wikidata += "=Maintainer Notes=\n"
174 if app.MaintainerNotes:
175 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
176 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
178 # Get a list of all packages for this application...
180 gotcurrentver = False
184 if apk['packageName'] == appid:
185 if str(apk['versionCode']) == app.CurrentVersionCode:
188 # Include ones we can't build, as a special case...
189 for build in app.builds:
191 if build.versionCode == app.CurrentVersionCode:
193 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
194 apklist.append({'versionCode': int(build.versionCode),
195 'versionName': build.versionName,
196 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
201 if apk['versionCode'] == int(build.versionCode):
206 apklist.append({'versionCode': int(build.versionCode),
207 'versionName': build.versionName,
208 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
210 if app.CurrentVersionCode == '0':
212 # Sort with most recent first...
213 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
215 wikidata += "=Versions=\n"
216 if len(apklist) == 0:
217 wikidata += "We currently have no versions of this app available."
218 elif not gotcurrentver:
219 wikidata += "We don't have the current version of this app."
221 wikidata += "We have the current version of this app."
222 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
223 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
224 if len(app.NoSourceSince) > 0:
225 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
226 if len(app.CurrentVersion) > 0:
227 wikidata += "The current (recommended) version is " + app.CurrentVersion
228 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
231 wikidata += "==" + apk['versionName'] + "==\n"
233 if 'buildproblem' in apk:
234 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
237 wikidata += "This version is built and signed by "
239 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
241 wikidata += "the original developer.\n\n"
242 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
244 wikidata += '\n[[Category:' + wikicat + ']]\n'
245 if len(app.NoSourceSince) > 0:
246 wikidata += '\n[[Category:Apps missing source code]]\n'
247 if validapks == 0 and not app.Disabled:
248 wikidata += '\n[[Category:Apps with no packages]]\n'
249 if cantupdate and not app.Disabled:
250 wikidata += "\n[[Category:Apps we cannot update]]\n"
251 if buildfails and not app.Disabled:
252 wikidata += "\n[[Category:Apps with failing builds]]\n"
253 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
254 wikidata += '\n[[Category:Apps to Update]]\n'
256 wikidata += '\n[[Category:Apps that are disabled]]\n'
257 if app.UpdateCheckMode == 'None' and not app.Disabled:
258 wikidata += '\n[[Category:Apps with no update check]]\n'
259 for appcat in app.Categories:
260 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
262 # We can't have underscores in the page name, even if they're in
263 # the package ID, because MediaWiki messes with them...
264 pagename = appid.replace('_', ' ')
266 # Drop a trailing newline, because mediawiki is going to drop it anyway
267 # and it we don't we'll think the page has changed when it hasn't...
268 if wikidata.endswith('\n'):
269 wikidata = wikidata[:-1]
271 generated_pages[pagename] = wikidata
273 # Make a redirect from the name to the ID too, unless there's
274 # already an existing page with the name and it isn't a redirect.
276 apppagename = app.Name.replace('_', ' ')
277 apppagename = apppagename.replace('{', '')
278 apppagename = apppagename.replace('}', ' ')
279 apppagename = apppagename.replace(':', ' ')
280 apppagename = apppagename.replace('[', ' ')
281 apppagename = apppagename.replace(']', ' ')
282 # Drop double spaces caused mostly by replacing ':' above
283 apppagename = apppagename.replace(' ', ' ')
284 for expagename in site.allpages(prefix=apppagename,
285 filterredir='nonredirects',
287 if expagename == apppagename:
289 # Another reason not to make the redirect page is if the app name
290 # is the same as it's ID, because that will overwrite the real page
291 # with an redirect to itself! (Although it seems like an odd
292 # scenario this happens a lot, e.g. where there is metadata but no
293 # builds or binaries to extract a name from.
294 if apppagename == pagename:
297 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
299 for tcat, genp in [(wikicat, generated_pages),
300 (wikiredircat, generated_redirects)]:
301 catpages = site.Pages['Category:' + tcat]
303 for page in catpages:
304 existingpages.append(page.name)
305 if page.name in genp:
306 pagetxt = page.edit()
307 if pagetxt != genp[page.name]:
308 logging.debug("Updating modified page " + page.name)
309 page.save(genp[page.name], summary='Auto-updated')
311 logging.debug("Page " + page.name + " is unchanged")
313 logging.warn("Deleting page " + page.name)
314 page.delete('No longer published')
315 for pagename, text in genp.items():
316 logging.debug("Checking " + pagename)
317 if pagename not in existingpages:
318 logging.debug("Creating page " + pagename)
320 newpage = site.Pages[pagename]
321 newpage.save(text, summary='Auto-created')
322 except Exception as e:
323 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
325 # Purge server cache to ensure counts are up to date
326 site.Pages['Repository Maintenance'].purge()
328 # Write a page with the last build log for this version code
329 wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
330 newpage = site.Pages[wiki_page_path]
332 txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
333 txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
334 txt += "* completed at " + common.get_wiki_timestamp() + '\n'
336 txt += common.get_android_tools_version_log()
337 newpage.save(txt, summary='Run log')
338 newpage = site.Pages['update']
339 newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
342 def delete_disabled_builds(apps, apkcache, repodirs):
343 """Delete disabled build outputs.
345 :param apps: list of all applications, as per metadata.read_metadata
346 :param apkcache: current apk cache information
347 :param repodirs: the repo directories to process
349 for appid, app in apps.items():
350 for build in app['builds']:
351 if not build.disable:
353 apkfilename = common.get_release_filename(app, build)
354 iconfilename = "%s.%s.png" % (
357 for repodir in repodirs:
359 os.path.join(repodir, apkfilename),
360 os.path.join(repodir, apkfilename + '.asc'),
361 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
363 for density in all_screen_densities:
364 repo_dir = get_icon_dir(repodir, density)
365 files.append(os.path.join(repo_dir, iconfilename))
368 if os.path.exists(f):
369 logging.info("Deleting disabled build output " + f)
371 if apkfilename in apkcache:
372 del apkcache[apkfilename]
375 def resize_icon(iconpath, density):
377 if not os.path.isfile(iconpath):
382 fp = open(iconpath, 'rb')
384 size = dpi_to_px(density)
386 if any(length > size for length in im.size):
388 im.thumbnail((size, size), Image.ANTIALIAS)
389 logging.debug("%s was too large at %s - new size is %s" % (
390 iconpath, oldsize, im.size))
391 im.save(iconpath, "PNG", optimize=True,
392 pnginfo=BLANK_PNG_INFO, icc_profile=None)
394 except Exception as e:
395 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
402 def resize_all_icons(repodirs):
403 """Resize all icons that exceed the max size
405 :param repodirs: the repo directories to process
407 for repodir in repodirs:
408 for density in screen_densities:
409 icon_dir = get_icon_dir(repodir, density)
410 icon_glob = os.path.join(icon_dir, '*.png')
411 for iconpath in glob.glob(icon_glob):
412 resize_icon(iconpath, density)
416 """ Get the signing certificate of an apk. To get the same md5 has that
417 Android gets, we encode the .RSA certificate in a specific format and pass
418 it hex-encoded to the md5 digest algorithm.
420 :param apkpath: path to the apk
421 :returns: A string containing the md5 of the signature of the apk or None
422 if an error occurred.
425 with zipfile.ZipFile(apkpath, 'r') as apk:
426 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
429 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
432 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
435 cert = apk.read(certs[0])
437 cert_encoded = common.get_certificate(cert)
439 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
442 def get_cache_file():
443 return os.path.join('tmp', 'apkcache')
447 """Get the cached dict of the APK index
449 Gather information about all the apk files in the repo directory,
450 using cached data if possible. Some of the index operations take a
451 long time, like calculating the SHA-256 and verifying the APK
454 The cache is invalidated if the metadata version is different, or
455 the 'allow_disabled_algorithms' config/option is different. In
456 those cases, there is no easy way to know what has changed from
457 the cache, so just rerun the whole thing.
462 apkcachefile = get_cache_file()
463 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
464 if not options.clean and os.path.exists(apkcachefile):
465 with open(apkcachefile, 'rb') as cf:
466 apkcache = pickle.load(cf, encoding='utf-8')
467 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
468 or apkcache.get('allow_disabled_algorithms') != ada:
473 apkcache["METADATA_VERSION"] = METADATA_VERSION
474 apkcache['allow_disabled_algorithms'] = ada
479 def write_cache(apkcache):
480 apkcachefile = get_cache_file()
481 cache_path = os.path.dirname(apkcachefile)
482 if not os.path.exists(cache_path):
483 os.makedirs(cache_path)
484 with open(apkcachefile, 'wb') as cf:
485 pickle.dump(apkcache, cf)
488 def get_icon_bytes(apkzip, iconsrc):
489 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
491 return apkzip.read(iconsrc)
493 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
496 def sha256sum(filename):
497 '''Calculate the sha256 of the given file'''
498 sha = hashlib.sha256()
499 with open(filename, 'rb') as f:
505 return sha.hexdigest()
508 def has_known_vulnerability(filename):
509 """checks for known vulnerabilities in the APK
511 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
512 version. Google also enforces this:
513 https://support.google.com/faqs/answer/6376725?hl=en
515 Checks whether there are more than one classes.dex or AndroidManifest.xml
516 files, which is invalid and an essential part of the "Master Key" attack.
517 http://www.saurik.com/id/17
519 Janus is similar to Master Key but is perhaps easier to scan for.
520 https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
525 # statically load this pattern
526 if not hasattr(has_known_vulnerability, "pattern"):
527 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
529 with open(filename.encode(), 'rb') as fp:
531 if first4 != b'\x50\x4b\x03\x04':
532 raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
533 .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
534 + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
537 with zipfile.ZipFile(filename) as zf:
538 for name in zf.namelist():
539 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
542 chunk = lib.read(4096)
545 m = has_known_vulnerability.pattern.search(chunk)
547 version = m.group(1).decode('ascii')
548 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
549 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
550 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
551 logging.debug(_('"{path}" contains recent {name} ({version})')
552 .format(path=filename, name=name, version=version))
554 logging.warning(_('"{path}" contains outdated {name} ({version})')
555 .format(path=filename, name=name, version=version))
558 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
559 if name in files_in_apk:
560 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
561 .format(apkfilename=filename, name=name))
563 files_in_apk.add(name)
567 def insert_obbs(repodir, apps, apks):
568 """Scans the .obb files in a given repo directory and adds them to the
569 relevant APK instances. OBB files have versionCodes like APK
570 files, and they are loosely associated. If there is an OBB file
571 present, then any APK with the same or higher versionCode will use
572 that OBB file. There are two OBB types: main and patch, each APK
573 can only have only have one of each.
575 https://developer.android.com/google/play/expansion-files.html
577 :param repodir: repo directory to scan
578 :param apps: list of current, valid apps
579 :param apks: current information on all APKs
583 def obbWarnDelete(f, msg):
584 logging.warning(msg + ' ' + f)
585 if options.delete_unknown:
586 logging.error(_("Deleting unknown file: {path}").format(path=f))
590 java_Integer_MIN_VALUE = -pow(2, 31)
591 currentPackageNames = apps.keys()
592 for f in glob.glob(os.path.join(repodir, '*.obb')):
593 obbfile = os.path.basename(f)
594 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
595 chunks = obbfile.split('.')
596 if chunks[0] != 'main' and chunks[0] != 'patch':
597 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
599 if not re.match(r'^-?[0-9]+$', chunks[1]):
600 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
601 .format(name=chunks[0]))
603 versionCode = int(chunks[1])
604 packagename = ".".join(chunks[2:-1])
606 highestVersionCode = java_Integer_MIN_VALUE
607 if packagename not in currentPackageNames:
608 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
611 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
612 highestVersionCode = apk['versionCode']
613 if versionCode > highestVersionCode:
614 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
615 .format(integer=str(versionCode)))
617 obbsha256 = sha256sum(f)
618 obbs.append((packagename, versionCode, obbfile, obbsha256))
621 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
622 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
623 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
624 apk['obbMainFile'] = obbfile
625 apk['obbMainFileSha256'] = obbsha256
626 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
627 apk['obbPatchFile'] = obbfile
628 apk['obbPatchFileSha256'] = obbsha256
629 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
633 def translate_per_build_anti_features(apps, apks):
634 """Grab the anti-features list from the build metadata
636 For most Anti-Features, they are really most applicable per-APK,
637 not for an app. An app can fix a vulnerability, add/remove
638 tracking, etc. This reads the 'antifeatures' list from the Build
639 entries in the fdroiddata metadata file, then transforms it into
640 the 'antiFeatures' list of unique items for the index.
642 The field key is all lower case in the metadata file to match the
643 rest of the Build fields. It is 'antiFeatures' camel case in the
644 implementation, index, and fdroidclient since it is translated
645 from the build 'antifeatures' field, not directly included.
649 antiFeatures = dict()
650 for packageName, app in apps.items():
652 for build in app['builds']:
653 afl = build.get('antifeatures')
655 d[int(build.versionCode)] = afl
657 antiFeatures[packageName] = d
660 d = antiFeatures.get(apk['packageName'])
662 afl = d.get(apk['versionCode'])
664 apk['antiFeatures'].update(afl)
667 def _get_localized_dict(app, locale):
668 '''get the dict to add localized store metadata to'''
669 if 'localized' not in app:
670 app['localized'] = collections.OrderedDict()
671 if locale not in app['localized']:
672 app['localized'][locale] = collections.OrderedDict()
673 return app['localized'][locale]
676 def _set_localized_text_entry(app, locale, key, f):
677 limit = config['char_limits'][key]
678 localized = _get_localized_dict(app, locale)
680 text = fp.read()[:limit]
682 localized[key] = text
685 def _set_author_entry(app, key, f):
686 limit = config['char_limits']['author']
688 text = fp.read()[:limit]
693 def _strip_and_copy_image(inpath, outpath):
694 """Remove any metadata from image and copy it to new path
696 Sadly, image metadata like EXIF can be used to exploit devices.
697 It is not used at all in the F-Droid ecosystem, so its much safer
698 just to remove it entirely.
702 extension = common.get_extension(inpath)[1]
703 if os.path.isdir(outpath):
704 outpath = os.path.join(outpath, os.path.basename(inpath))
705 if extension == 'png':
706 with open(inpath, 'rb') as fp:
707 in_image = Image.open(fp)
708 in_image.save(outpath, "PNG", optimize=True,
709 pnginfo=BLANK_PNG_INFO, icc_profile=None)
710 elif extension == 'jpg' or extension == 'jpeg':
711 with open(inpath, 'rb') as fp:
712 in_image = Image.open(fp)
713 data = list(in_image.getdata())
714 out_image = Image.new(in_image.mode, in_image.size)
715 out_image.putdata(data)
716 out_image.save(outpath, "JPEG", optimize=True)
718 raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
719 .format(extension=extension))
722 def copy_triple_t_store_metadata(apps):
723 """Include store metadata from the app's source repo
725 The Triple-T Gradle Play Publisher is a plugin that has a standard
726 file layout for all of the metadata and graphics that the Google
727 Play Store accepts. Since F-Droid has the git repo, it can just
728 pluck those files directly. This method reads any text files into
729 the app dict, then copies any graphics into the fdroid repo
732 This needs to be run before insert_localized_app_metadata() so that
733 the graphics files that are copied into the fdroid repo get
736 https://github.com/Triple-T/gradle-play-publisher#upload-images
737 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
741 if not os.path.isdir('build'):
742 return # nothing to do
744 for packageName, app in apps.items():
745 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
746 logging.debug('Triple-T Gradle Play Publisher: ' + d)
747 for root, dirs, files in os.walk(d):
748 segments = root.split('/')
749 locale = segments[-2]
751 if f == 'fulldescription':
752 _set_localized_text_entry(app, locale, 'description',
753 os.path.join(root, f))
755 elif f == 'shortdescription':
756 _set_localized_text_entry(app, locale, 'summary',
757 os.path.join(root, f))
760 _set_localized_text_entry(app, locale, 'name',
761 os.path.join(root, f))
764 _set_localized_text_entry(app, locale, 'video',
765 os.path.join(root, f))
767 elif f == 'whatsnew':
768 _set_localized_text_entry(app, segments[-1], 'whatsNew',
769 os.path.join(root, f))
771 elif f == 'contactEmail':
772 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
774 elif f == 'contactPhone':
775 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
777 elif f == 'contactWebsite':
778 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
781 base, extension = common.get_extension(f)
782 dirname = os.path.basename(root)
783 if extension in ALLOWED_EXTENSIONS \
784 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
785 if segments[-2] == 'listing':
786 locale = segments[-3]
788 locale = segments[-2]
789 destdir = os.path.join('repo', packageName, locale, dirname)
790 os.makedirs(destdir, mode=0o755, exist_ok=True)
791 sourcefile = os.path.join(root, f)
792 destfile = os.path.join(destdir, os.path.basename(f))
793 logging.debug('copying ' + sourcefile + ' ' + destfile)
794 _strip_and_copy_image(sourcefile, destfile)
797 def insert_localized_app_metadata(apps):
798 """scans standard locations for graphics and localized text
800 Scans for localized description files, store graphics, and
801 screenshot PNG files in statically defined screenshots directory
802 and adds them to the app metadata. The screenshots and graphic
803 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
804 and must be in the following layout:
805 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
807 repo/packageName/locale/featureGraphic.png
808 repo/packageName/locale/phoneScreenshots/1.png
809 repo/packageName/locale/phoneScreenshots/2.png
811 The changelog files must be text files named with the versionCode
812 ending with ".txt" and must be in the following layout:
813 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
815 repo/packageName/locale/changelogs/12345.txt
817 This will scan the each app's source repo then the metadata/ dir
818 for these standard locations of changelog files. If it finds
819 them, they will be added to the dict of all packages, with the
820 versions in the metadata/ folder taking precendence over the what
821 is in the app's source repo.
823 Where "packageName" is the app's packageName and "locale" is the locale
824 of the graphics, e.g. what language they are in, using the IETF RFC5646
825 format (en-US, fr-CA, es-MX, etc).
827 This will also scan the app's git for a fastlane folder, and the
828 metadata/ folder and the apps' source repos for standard locations
829 of graphic and screenshot files. If it finds them, it will copy
830 them into the repo. The fastlane files follow this pattern:
831 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
835 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
836 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
837 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
838 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
840 for srcd in sorted(sourcedirs):
841 if not os.path.isdir(srcd):
843 for root, dirs, files in os.walk(srcd):
844 segments = root.split('/')
845 packageName = segments[1]
846 if packageName not in apps:
847 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
849 locale = segments[-1]
850 destdir = os.path.join('repo', packageName, locale)
852 # flavours specified in build receipt
854 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
855 and 'gradle' in apps[packageName].builds[-1]:
856 build_flavours = apps[packageName].builds[-1].gradle
858 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
859 logging.debug("ignoring due to wrong flavour")
863 if f in ('description.txt', 'full_description.txt'):
864 _set_localized_text_entry(apps[packageName], locale, 'description',
865 os.path.join(root, f))
867 elif f in ('summary.txt', 'short_description.txt'):
868 _set_localized_text_entry(apps[packageName], locale, 'summary',
869 os.path.join(root, f))
871 elif f in ('name.txt', 'title.txt'):
872 _set_localized_text_entry(apps[packageName], locale, 'name',
873 os.path.join(root, f))
875 elif f == 'video.txt':
876 _set_localized_text_entry(apps[packageName], locale, 'video',
877 os.path.join(root, f))
879 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
880 locale = segments[-2]
881 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
882 os.path.join(root, f))
885 base, extension = common.get_extension(f)
886 if locale == 'images':
887 locale = segments[-2]
888 destdir = os.path.join('repo', packageName, locale)
889 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
890 os.makedirs(destdir, mode=0o755, exist_ok=True)
891 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
892 _strip_and_copy_image(os.path.join(root, f), destdir)
894 if d in SCREENSHOT_DIRS:
895 if locale == 'images':
896 locale = segments[-2]
897 destdir = os.path.join('repo', packageName, locale)
898 for f in glob.glob(os.path.join(root, d, '*.*')):
899 _ignored, extension = common.get_extension(f)
900 if extension in ALLOWED_EXTENSIONS:
901 screenshotdestdir = os.path.join(destdir, d)
902 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
903 logging.debug('copying ' + f + ' ' + screenshotdestdir)
904 _strip_and_copy_image(f, screenshotdestdir)
906 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
908 if not os.path.isdir(d):
910 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
911 if not os.path.isfile(f):
913 segments = f.split('/')
914 packageName = segments[1]
916 screenshotdir = segments[3]
917 filename = os.path.basename(f)
918 base, extension = common.get_extension(filename)
920 if packageName not in apps:
921 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
922 .format(path=filename, name=packageName))
924 graphics = _get_localized_dict(apps[packageName], locale)
926 if extension not in ALLOWED_EXTENSIONS:
927 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
928 elif base in GRAPHIC_NAMES:
929 # there can only be zero or one of these per locale
930 graphics[base] = filename
931 elif screenshotdir in SCREENSHOT_DIRS:
932 # there can any number of these per locale
933 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
934 if screenshotdir not in graphics:
935 graphics[screenshotdir] = []
936 graphics[screenshotdir].append(filename)
938 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
941 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
942 """Scan a repo for all files with an extension except APK/OBB
944 :param apkcache: current cached info about all repo files
945 :param repodir: repo directory to scan
946 :param knownapks: list of all known files, as per metadata.read_metadata
947 :param use_date_from_file: use date from file (instead of current date)
948 for newly added files
953 repodir = repodir.encode('utf-8')
954 for name in os.listdir(repodir):
955 file_extension = common.get_file_extension(name)
956 if file_extension == 'apk' or file_extension == 'obb':
958 filename = os.path.join(repodir, name)
959 name_utf8 = name.decode('utf-8')
960 if filename.endswith(b'_src.tar.gz'):
961 logging.debug(_('skipping source tarball: {path}')
962 .format(path=filename.decode('utf-8')))
964 if not common.is_repo_file(filename):
966 stat = os.stat(filename)
967 if stat.st_size == 0:
968 raise FDroidException(_('{path} is zero size!')
969 .format(path=filename))
971 shasum = sha256sum(filename)
974 repo_file = apkcache[name]
975 # added time is cached as tuple but used here as datetime instance
976 if 'added' in repo_file:
977 a = repo_file['added']
978 if isinstance(a, datetime):
979 repo_file['added'] = a
981 repo_file['added'] = datetime(*a[:6])
982 if repo_file.get('hash') == shasum:
983 logging.debug(_("Reading {apkfilename} from cache")
984 .format(apkfilename=name_utf8))
987 logging.debug(_("Ignoring stale cache data for {apkfilename}")
988 .format(apkfilename=name_utf8))
991 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
992 repo_file = collections.OrderedDict()
993 repo_file['name'] = os.path.splitext(name_utf8)[0]
994 # TODO rename apkname globally to something more generic
995 repo_file['apkName'] = name_utf8
996 repo_file['hash'] = shasum
997 repo_file['hashType'] = 'sha256'
998 repo_file['versionCode'] = 0
999 repo_file['versionName'] = shasum
1000 # the static ID is the SHA256 unless it is set in the metadata
1001 repo_file['packageName'] = shasum
1003 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1005 repo_file['packageName'] = m.group(1)
1006 repo_file['versionCode'] = int(m.group(2))
1007 srcfilename = name + b'_src.tar.gz'
1008 if os.path.exists(os.path.join(repodir, srcfilename)):
1009 repo_file['srcname'] = srcfilename.decode('utf-8')
1010 repo_file['size'] = stat.st_size
1012 apkcache[name] = repo_file
1015 if use_date_from_file:
1016 timestamp = stat.st_ctime
1017 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1019 default_date_param = None
1021 # Record in knownapks, getting the added date at the same time..
1022 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1023 default_date=default_date_param)
1025 repo_file['added'] = added
1027 repo_files.append(repo_file)
1029 return repo_files, cachechanged
1032 def scan_apk(apk_file):
1034 Scans an APK file and returns dictionary with metadata of the APK.
1036 Attention: This does *not* verify that the APK signature is correct.
1038 :param apk_file: The (ideally absolute) path to the APK file
1039 :raises BuildException
1040 :return A dict containing APK metadata
1043 'hash': sha256sum(apk_file),
1044 'hashType': 'sha256',
1045 'uses-permission': [],
1046 'uses-permission-sdk-23': [],
1050 'antiFeatures': set(),
1053 if SdkToolsPopen(['aapt', 'version'], output=False):
1054 scan_apk_aapt(apk, apk_file)
1056 scan_apk_androguard(apk, apk_file)
1058 # Get the signature, or rather the signing key fingerprints
1059 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1060 apk['sig'] = getsig(apk_file)
1062 raise BuildException("Failed to get apk signature")
1063 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1065 if not apk.get('signer'):
1066 raise BuildException("Failed to get apk signing key fingerprint")
1068 # Get size of the APK
1069 apk['size'] = os.path.getsize(apk_file)
1071 if 'minSdkVersion' not in apk:
1072 logging.warning("No SDK version information found in {0}".format(apk_file))
1073 apk['minSdkVersion'] = 1
1075 # Check for known vulnerabilities
1076 if has_known_vulnerability(apk_file):
1077 apk['antiFeatures'].add('KnownVuln')
1082 def scan_apk_aapt(apk, apkfile):
1083 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1084 if p.returncode != 0:
1085 if options.delete_unknown:
1086 if os.path.exists(apkfile):
1087 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1090 logging.error("Could not find {0} to remove it".format(apkfile))
1092 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1093 raise BuildException(_("Invalid APK"))
1094 for line in p.output.splitlines():
1095 if line.startswith("package:"):
1097 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1098 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1099 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1100 except Exception as e:
1101 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1102 elif line.startswith("application:"):
1103 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1104 # Keep path to non-dpi icon in case we need it
1105 match = re.match(APK_ICON_PAT_NODPI, line)
1107 apk['icons_src']['-1'] = match.group(1)
1108 elif line.startswith("launchable-activity:"):
1109 # Only use launchable-activity as fallback to application
1111 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1112 if '-1' not in apk['icons_src']:
1113 match = re.match(APK_ICON_PAT_NODPI, line)
1115 apk['icons_src']['-1'] = match.group(1)
1116 elif line.startswith("application-icon-"):
1117 match = re.match(APK_ICON_PAT, line)
1119 density = match.group(1)
1120 path = match.group(2)
1121 apk['icons_src'][density] = path
1122 elif line.startswith("sdkVersion:"):
1123 m = re.match(APK_SDK_VERSION_PAT, line)
1125 logging.error(line.replace('sdkVersion:', '')
1126 + ' is not a valid minSdkVersion!')
1128 apk['minSdkVersion'] = m.group(1)
1129 # if target not set, default to min
1130 if 'targetSdkVersion' not in apk:
1131 apk['targetSdkVersion'] = m.group(1)
1132 elif line.startswith("targetSdkVersion:"):
1133 m = re.match(APK_SDK_VERSION_PAT, line)
1135 logging.error(line.replace('targetSdkVersion:', '')
1136 + ' is not a valid targetSdkVersion!')
1138 apk['targetSdkVersion'] = m.group(1)
1139 elif line.startswith("maxSdkVersion:"):
1140 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1141 elif line.startswith("native-code:"):
1142 apk['nativecode'] = []
1143 for arch in line[13:].split(' '):
1144 apk['nativecode'].append(arch[1:-1])
1145 elif line.startswith('uses-permission:'):
1146 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1147 if perm_match['maxSdkVersion']:
1148 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1149 permission = UsesPermission(
1151 perm_match['maxSdkVersion']
1154 apk['uses-permission'].append(permission)
1155 elif line.startswith('uses-permission-sdk-23:'):
1156 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1157 if perm_match['maxSdkVersion']:
1158 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1159 permission_sdk_23 = UsesPermissionSdk23(
1161 perm_match['maxSdkVersion']
1164 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1166 elif line.startswith('uses-feature:'):
1167 feature = re.match(APK_FEATURE_PAT, line).group(1)
1168 # Filter out this, it's only added with the latest SDK tools and
1169 # causes problems for lots of apps.
1170 if feature != "android.hardware.screen.portrait" \
1171 and feature != "android.hardware.screen.landscape":
1172 if feature.startswith("android.feature."):
1173 feature = feature[16:]
1174 apk['features'].add(feature)
1177 def scan_apk_androguard(apk, apkfile):
1179 from androguard.core.bytecodes.apk import APK
1180 apkobject = APK(apkfile)
1181 if apkobject.is_valid_APK():
1182 arsc = apkobject.get_android_resources()
1184 if options.delete_unknown:
1185 if os.path.exists(apkfile):
1186 logging.error(_("Failed to get apk information, deleting {path}")
1187 .format(path=apkfile))
1190 logging.error(_("Could not find {path} to remove it")
1191 .format(path=apkfile))
1193 logging.error(_("Failed to get apk information, skipping {path}")
1194 .format(path=apkfile))
1195 raise BuildException(_("Invalid APK"))
1197 raise FDroidException("androguard library is not installed and aapt not present")
1198 except FileNotFoundError:
1199 logging.error(_("Could not open apk file for analysis"))
1200 raise BuildException(_("Invalid APK"))
1202 apk['packageName'] = apkobject.get_package()
1203 apk['versionCode'] = int(apkobject.get_androidversion_code())
1204 apk['versionName'] = apkobject.get_androidversion_name()
1205 if apk['versionName'][0] == "@":
1206 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1207 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1208 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1209 apk['name'] = apkobject.get_app_name()
1211 if apkobject.get_max_sdk_version() is not None:
1212 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1213 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1214 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1216 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1217 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1219 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1221 for file in apkobject.get_files():
1222 d_re = density_re.match(file)
1224 folder = d_re.group(1).split('-')
1226 resolution = folder[1]
1229 density = screen_resolutions[resolution]
1230 apk['icons_src'][density] = d_re.group(0)
1232 if apk['icons_src'].get('-1') is None:
1233 apk['icons_src']['-1'] = apk['icons_src']['160']
1235 arch_re = re.compile("^lib/(.*)/.*$")
1236 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1238 apk['nativecode'] = []
1239 apk['nativecode'].extend(sorted(list(arch)))
1241 xml = apkobject.get_android_manifest_xml()
1243 for item in xml.getElementsByTagName('uses-permission'):
1244 name = str(item.getAttribute("android:name"))
1245 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1246 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1247 permission = UsesPermission(
1251 apk['uses-permission'].append(permission)
1253 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1254 name = str(item.getAttribute("android:name"))
1255 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1256 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1257 permission_sdk_23 = UsesPermissionSdk23(
1261 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1263 for item in xml.getElementsByTagName('uses-feature'):
1264 feature = str(item.getAttribute("android:name"))
1265 if feature != "android.hardware.screen.portrait" \
1266 and feature != "android.hardware.screen.landscape":
1267 if feature.startswith("android.feature."):
1268 feature = feature[16:]
1269 apk['features'].append(feature)
1272 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1273 allow_disabled_algorithms=False, archive_bad_sig=False):
1274 """Processes the apk with the given filename in the given repo directory.
1276 This also extracts the icons.
1278 :param apkcache: current apk cache information
1279 :param apkfilename: the filename of the apk to scan
1280 :param repodir: repo directory to scan
1281 :param knownapks: known apks info
1282 :param use_date_from_apk: use date from APK (instead of current date)
1283 for newly added APKs
1284 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1285 disabled algorithms in the signature (e.g. MD5)
1286 :param archive_bad_sig: move APKs with a bad signature to the archive
1287 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1288 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1292 apkfile = os.path.join(repodir, apkfilename)
1294 cachechanged = False
1296 if apkfilename in apkcache:
1297 apk = apkcache[apkfilename]
1298 if apk.get('hash') == sha256sum(apkfile):
1299 logging.debug(_("Reading {apkfilename} from cache")
1300 .format(apkfilename=apkfilename))
1303 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1304 .format(apkfilename=apkfilename))
1307 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1310 apk = scan_apk(apkfile)
1311 except BuildException:
1312 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1313 .format(apkfilename=apkfilename))
1314 return True, None, False
1316 # Check for debuggable apks...
1317 if common.isApkAndDebuggable(apkfile):
1318 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1320 if options.rename_apks:
1321 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1322 std_short_name = os.path.join(repodir, n)
1323 if apkfile != std_short_name:
1324 if os.path.exists(std_short_name):
1325 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1326 if apkfile != std_long_name:
1327 if os.path.exists(std_long_name):
1328 dupdir = os.path.join('duplicates', repodir)
1329 if not os.path.isdir(dupdir):
1330 os.makedirs(dupdir, exist_ok=True)
1331 dupfile = os.path.join('duplicates', std_long_name)
1332 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1333 os.rename(apkfile, dupfile)
1334 return True, None, False
1336 os.rename(apkfile, std_long_name)
1337 apkfile = std_long_name
1339 os.rename(apkfile, std_short_name)
1340 apkfile = std_short_name
1341 apkfilename = apkfile[len(repodir) + 1:]
1343 apk['apkName'] = apkfilename
1344 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1345 if os.path.exists(os.path.join(repodir, srcfilename)):
1346 apk['srcname'] = srcfilename
1348 # verify the jar signature is correct, allow deprecated
1349 # algorithms only if the APK is in the archive.
1351 if not common.verify_apk_signature(apkfile):
1352 if repodir == 'archive' or allow_disabled_algorithms:
1353 if common.verify_old_apk_signature(apkfile):
1354 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1362 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1363 .format(apkfilename=apkfilename))
1364 move_apk_between_sections(repodir, 'archive', apk)
1366 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1367 .format(apkfilename=apkfilename))
1368 return True, None, False
1370 apkzip = zipfile.ZipFile(apkfile, 'r')
1372 manifest = apkzip.getinfo('AndroidManifest.xml')
1373 # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1374 if (1980, 0, 0) != manifest.date_time[0:3]:
1376 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1377 except ValueError as e:
1378 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1379 .format(apkfilename=apkfile) + str(e))
1381 # extract icons from APK zip file
1382 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1384 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1386 apkzip.close() # ensure that APK zip file gets closed
1388 # resize existing icons for densities missing in the APK
1389 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1391 if use_date_from_apk and manifest.date_time[1] != 0:
1392 default_date_param = datetime(*manifest.date_time)
1394 default_date_param = None
1396 # Record in known apks, getting the added date at the same time..
1397 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1398 default_date=default_date_param)
1400 apk['added'] = added
1402 apkcache[apkfilename] = apk
1405 return False, apk, cachechanged
1408 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1409 """Processes the apks in the given repo directory.
1411 This also extracts the icons.
1413 :param apkcache: current apk cache information
1414 :param repodir: repo directory to scan
1415 :param knownapks: known apks info
1416 :param use_date_from_apk: use date from APK (instead of current date)
1417 for newly added APKs
1418 :returns: (apks, cachechanged) where apks is a list of apk information,
1419 and cachechanged is True if the apkcache got changed.
1422 cachechanged = False
1424 for icon_dir in get_all_icon_dirs(repodir):
1425 if os.path.exists(icon_dir):
1427 shutil.rmtree(icon_dir)
1428 os.makedirs(icon_dir)
1430 os.makedirs(icon_dir)
1433 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1434 apkfilename = apkfile[len(repodir) + 1:]
1435 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1436 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1437 use_date_from_apk, ada, True)
1441 cachechanged = cachechanged or cachethis
1443 return apks, cachechanged
1446 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1448 Extracts icons from the given APK zip in various densities,
1449 saves them into given repo directory
1450 and stores their names in the APK metadata dictionary.
1452 :param icon_filename: A string representing the icon's file name
1453 :param apk: A populated dictionary containing APK metadata.
1454 Needs to have 'icons_src' key
1455 :param apkzip: An opened zipfile.ZipFile of the APK file
1456 :param repo_dir: The directory of the APK's repository
1457 :return: A list of icon densities that are missing
1459 empty_densities = []
1460 for density in screen_densities:
1461 if density not in apk['icons_src']:
1462 empty_densities.append(density)
1464 icon_src = apk['icons_src'][density]
1465 icon_dir = get_icon_dir(repo_dir, density)
1466 icon_dest = os.path.join(icon_dir, icon_filename)
1468 # Extract the icon files per density
1469 if icon_src.endswith('.xml'):
1470 png = os.path.basename(icon_src)[:-4] + '.png'
1471 for f in apkzip.namelist():
1473 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1474 if m and screen_resolutions[m.group(2)] == density:
1476 if icon_src.endswith('.xml'):
1477 empty_densities.append(density)
1480 with open(icon_dest, 'wb') as f:
1481 f.write(get_icon_bytes(apkzip, icon_src))
1482 apk['icons'][density] = icon_filename
1483 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1484 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1485 del apk['icons_src'][density]
1486 empty_densities.append(density)
1488 if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1489 icon_src = apk['icons_src']['-1']
1490 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1491 with open(icon_path, 'wb') as f:
1492 f.write(get_icon_bytes(apkzip, icon_src))
1495 im = Image.open(icon_path)
1496 dpi = px_to_dpi(im.size[0])
1497 for density in screen_densities:
1498 if density in apk['icons']:
1500 if density == screen_densities[-1] or dpi >= int(density):
1501 apk['icons'][density] = icon_filename
1502 shutil.move(icon_path,
1503 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1504 empty_densities.remove(density)
1506 except Exception as e:
1507 logging.warning(_("Failed reading {path}: {error}")
1508 .format(path=icon_path, error=e))
1510 if im and hasattr(im, 'close'):
1514 apk['icon'] = icon_filename
1516 return empty_densities
1519 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1521 Resize existing icons for densities missing in the APK to ensure all densities are available
1523 :param empty_densities: A list of icon densities that are missing
1524 :param icon_filename: A string representing the icon's file name
1525 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1526 :param repo_dir: The directory of the APK's repository
1528 # First try resizing down to not lose quality
1530 for density in screen_densities:
1531 if density not in empty_densities:
1532 last_density = density
1534 if last_density is None:
1536 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1538 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1539 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1542 fp = open(last_icon_path, 'rb')
1545 size = dpi_to_px(density)
1547 im.thumbnail((size, size), Image.ANTIALIAS)
1548 im.save(icon_path, "PNG", optimize=True,
1549 pnginfo=BLANK_PNG_INFO, icc_profile=None)
1550 empty_densities.remove(density)
1551 except Exception as e:
1552 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1557 # Then just copy from the highest resolution available
1559 for density in reversed(screen_densities):
1560 if density not in empty_densities:
1561 last_density = density
1564 if last_density is None:
1568 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1569 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1571 empty_densities.remove(density)
1573 for density in screen_densities:
1574 icon_dir = get_icon_dir(repo_dir, density)
1575 icon_dest = os.path.join(icon_dir, icon_filename)
1576 resize_icon(icon_dest, density)
1578 # Copy from icons-mdpi to icons since mdpi is the baseline density
1579 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1580 if os.path.isfile(baseline):
1581 apk['icons']['0'] = icon_filename
1582 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1585 def apply_info_from_latest_apk(apps, apks):
1587 Some information from the apks needs to be applied up to the application level.
1588 When doing this, we use the info from the most recent version's apk.
1589 We deal with figuring out when the app was added and last updated at the same time.
1591 for appid, app in apps.items():
1592 bestver = UNSET_VERSION_CODE
1594 if apk['packageName'] == appid:
1595 if apk['versionCode'] > bestver:
1596 bestver = apk['versionCode']
1600 if not app.added or apk['added'] < app.added:
1601 app.added = apk['added']
1602 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1603 app.lastUpdated = apk['added']
1606 logging.debug("Don't know when " + appid + " was added")
1607 if not app.lastUpdated:
1608 logging.debug("Don't know when " + appid + " was last updated")
1610 if bestver == UNSET_VERSION_CODE:
1612 if app.Name is None:
1613 app.Name = app.AutoName or appid
1615 logging.debug("Application " + appid + " has no packages")
1617 if app.Name is None:
1618 app.Name = bestapk['name']
1619 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1620 if app.CurrentVersionCode is None:
1621 app.CurrentVersionCode = str(bestver)
1624 def make_categories_txt(repodir, categories):
1625 '''Write a category list in the repo to allow quick access'''
1627 for cat in sorted(categories):
1628 catdata += cat + '\n'
1629 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1633 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1635 def filter_apk_list_sorted(apk_list):
1637 for apk in apk_list:
1638 if apk['packageName'] == appid:
1641 # Sort the apk list by version code. First is highest/newest.
1642 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1644 for appid, app in apps.items():
1646 if app.ArchivePolicy:
1647 keepversions = int(app.ArchivePolicy[:-9])
1649 keepversions = defaultkeepversions
1651 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1652 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1654 current_app_apks = filter_apk_list_sorted(apks)
1655 if len(current_app_apks) > keepversions:
1656 # Move back the ones we don't want.
1657 for apk in current_app_apks[keepversions:]:
1658 move_apk_between_sections(repodir, archivedir, apk)
1659 archapks.append(apk)
1662 current_app_archapks = filter_apk_list_sorted(archapks)
1663 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1665 # Move forward the ones we want again, except DisableAlgorithm
1666 for apk in current_app_archapks:
1667 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1668 move_apk_between_sections(archivedir, repodir, apk)
1669 archapks.remove(apk)
1672 if kept == keepversions:
1676 def move_apk_between_sections(from_dir, to_dir, apk):
1677 """move an APK from repo to archive or vice versa"""
1679 def _move_file(from_dir, to_dir, filename, ignore_missing):
1680 from_path = os.path.join(from_dir, filename)
1681 if ignore_missing and not os.path.exists(from_path):
1683 to_path = os.path.join(to_dir, filename)
1684 if not os.path.exists(to_dir):
1686 shutil.move(from_path, to_path)
1688 if from_dir == to_dir:
1691 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1692 _move_file(from_dir, to_dir, apk['apkName'], False)
1693 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1694 for density in all_screen_densities:
1695 from_icon_dir = get_icon_dir(from_dir, density)
1696 to_icon_dir = get_icon_dir(to_dir, density)
1697 if density not in apk.get('icons', []):
1699 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1700 if 'srcname' in apk:
1701 _move_file(from_dir, to_dir, apk['srcname'], False)
1704 def add_apks_to_per_app_repos(repodir, apks):
1705 apks_per_app = dict()
1707 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1708 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1709 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1710 apks_per_app[apk['packageName']] = apk
1712 if not os.path.exists(apk['per_app_icons']):
1713 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1714 os.makedirs(apk['per_app_icons'])
1716 apkpath = os.path.join(repodir, apk['apkName'])
1717 shutil.copy(apkpath, apk['per_app_repo'])
1718 apksigpath = apkpath + '.sig'
1719 if os.path.exists(apksigpath):
1720 shutil.copy(apksigpath, apk['per_app_repo'])
1721 apkascpath = apkpath + '.asc'
1722 if os.path.exists(apkascpath):
1723 shutil.copy(apkascpath, apk['per_app_repo'])
1726 def create_metadata_from_template(apk):
1727 '''create a new metadata file using internal or external template
1729 Generate warnings for apk's with no metadata (or create skeleton
1730 metadata files, if requested on the command line). Though the
1731 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1732 since those impose things on the metadata file made from the
1733 template: field sort order, empty field value, formatting, etc.
1737 if os.path.exists('template.yml'):
1738 with open('template.yml') as f:
1740 if 'name' in apk and apk['name'] != '':
1741 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1742 r'\1 ' + apk['name'],
1744 flags=re.IGNORECASE | re.MULTILINE)
1746 logging.warning(_('{appid} does not have a name! Using package name instead.')
1747 .format(appid=apk['packageName']))
1748 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1749 r'\1 ' + apk['packageName'],
1751 flags=re.IGNORECASE | re.MULTILINE)
1752 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1756 app['Categories'] = [os.path.basename(os.getcwd())]
1757 # include some blanks as part of the template
1758 app['AuthorName'] = ''
1761 app['IssueTracker'] = ''
1762 app['SourceCode'] = ''
1763 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1764 if 'name' in apk and apk['name'] != '':
1765 app['Name'] = apk['name']
1767 logging.warning(_('{appid} does not have a name! Using package name instead.')
1768 .format(appid=apk['packageName']))
1769 app['Name'] = apk['packageName']
1770 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1771 yaml.dump(app, f, default_flow_style=False)
1772 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1777 start_timestamp = time.gmtime()
1782 global config, options
1784 # Parse command line...
1785 parser = ArgumentParser()
1786 common.setup_global_opts(parser)
1787 parser.add_argument("--create-key", action="store_true", default=False,
1788 help=_("Add a repo signing key to an unsigned repo"))
1789 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1790 help=_("Add skeleton metadata files for APKs that are missing them"))
1791 parser.add_argument("--delete-unknown", action="store_true", default=False,
1792 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1793 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1794 help=_("Report on build data status"))
1795 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1796 help=_("Interactively ask about things that need updating."))
1797 parser.add_argument("-I", "--icons", action="store_true", default=False,
1798 help=_("Resize all the icons exceeding the max pixel size and exit"))
1799 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1800 help=_("Specify editor to use in interactive mode. Default " +
1801 "is {path}").format(path='/etc/alternatives/editor'))
1802 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1803 help=_("Update the wiki"))
1804 parser.add_argument("--pretty", action="store_true", default=False,
1805 help=_("Produce human-readable XML/JSON for index files"))
1806 parser.add_argument("--clean", action="store_true", default=False,
1807 help=_("Clean update - don't uses caches, reprocess all APKs"))
1808 parser.add_argument("--nosign", action="store_true", default=False,
1809 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1810 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1811 help=_("Use date from APK instead of current time for newly added APKs"))
1812 parser.add_argument("--rename-apks", action="store_true", default=False,
1813 help=_("Rename APK files that do not match package.name_123.apk"))
1814 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1815 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1816 metadata.add_metadata_arguments(parser)
1817 options = parser.parse_args()
1818 metadata.warnings_action = options.W
1820 config = common.read_config(options)
1822 if not ('jarsigner' in config and 'keytool' in config):
1823 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1826 if config['archive_older'] != 0:
1827 repodirs.append('archive')
1828 if not os.path.exists('archive'):
1832 resize_all_icons(repodirs)
1835 if options.rename_apks:
1836 options.clean = True
1838 # check that icons exist now, rather than fail at the end of `fdroid update`
1839 for k in ['repo_icon', 'archive_icon']:
1841 if not os.path.exists(config[k]):
1842 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1843 .format(name=k, path=config[k]))
1846 # if the user asks to create a keystore, do it now, reusing whatever it can
1847 if options.create_key:
1848 if os.path.exists(config['keystore']):
1849 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1850 logging.critical("\t'" + config['keystore'] + "'")
1853 if 'repo_keyalias' not in config:
1854 config['repo_keyalias'] = socket.getfqdn()
1855 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1856 if 'keydname' not in config:
1857 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1858 common.write_to_config(config, 'keydname', config['keydname'])
1859 if 'keystore' not in config:
1860 config['keystore'] = common.default_config['keystore']
1861 common.write_to_config(config, 'keystore', config['keystore'])
1863 password = common.genpassword()
1864 if 'keystorepass' not in config:
1865 config['keystorepass'] = password
1866 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1867 if 'keypass' not in config:
1868 config['keypass'] = password
1869 common.write_to_config(config, 'keypass', config['keypass'])
1870 common.genkeystore(config)
1873 apps = metadata.read_metadata()
1875 # Generate a list of categories...
1877 for app in apps.values():
1878 categories.update(app.Categories)
1880 # Read known apks data (will be updated and written back when we've finished)
1881 knownapks = common.KnownApks()
1884 apkcache = get_cache()
1886 # Delete builds for disabled apps
1887 delete_disabled_builds(apps, apkcache, repodirs)
1889 # Scan all apks in the main repo
1890 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1892 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1893 options.use_date_from_apk)
1894 cachechanged = cachechanged or fcachechanged
1897 if apk['packageName'] not in apps:
1898 if options.create_metadata:
1899 create_metadata_from_template(apk)
1900 apps = metadata.read_metadata()
1902 msg = _("{apkfilename} ({appid}) has no metadata!") \
1903 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1904 if options.delete_unknown:
1905 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1906 .format(apkfilename=apk['apkName']))
1907 rmf = os.path.join(repodirs[0], apk['apkName'])
1908 if not os.path.exists(rmf):
1909 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1913 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1915 copy_triple_t_store_metadata(apps)
1916 insert_obbs(repodirs[0], apps, apks)
1917 insert_localized_app_metadata(apps)
1918 translate_per_build_anti_features(apps, apks)
1920 # Scan the archive repo for apks as well
1921 if len(repodirs) > 1:
1922 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1928 # Apply information from latest apks to the application and update dates
1929 apply_info_from_latest_apk(apps, apks + archapks)
1931 # Sort the app list by name, then the web site doesn't have to by default.
1932 # (we had to wait until we'd scanned the apks to do this, because mostly the
1933 # name comes from there!)
1934 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1936 # APKs are placed into multiple repos based on the app package, providing
1937 # per-app subscription feeds for nightly builds and things like it
1938 if config['per_app_repos']:
1939 add_apks_to_per_app_repos(repodirs[0], apks)
1940 for appid, app in apps.items():
1941 repodir = os.path.join(appid, 'fdroid', 'repo')
1943 appdict[appid] = app
1944 if os.path.isdir(repodir):
1945 index.make(appdict, [appid], apks, repodir, False)
1947 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1950 if len(repodirs) > 1:
1951 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1953 # Make the index for the main repo...
1954 index.make(apps, sortedids, apks, repodirs[0], False)
1955 make_categories_txt(repodirs[0], categories)
1957 # If there's an archive repo, make the index for it. We already scanned it
1959 if len(repodirs) > 1:
1960 index.make(apps, sortedids, archapks, repodirs[1], True)
1962 git_remote = config.get('binary_transparency_remote')
1963 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1965 btlog.make_binary_transparency_log(repodirs)
1967 if config['update_stats']:
1968 # Update known apks info...
1969 knownapks.writeifchanged()
1971 # Generate latest apps data for widget
1972 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1974 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1976 appid = line.rstrip()
1977 data += appid + "\t"
1979 data += app.Name + "\t"
1980 if app.icon is not None:
1981 data += app.icon + "\t"
1982 data += app.License + "\n"
1983 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1987 write_cache(apkcache)
1989 # Update the wiki...
1991 update_wiki(apps, sortedids, apks + archapks)
1993 logging.info(_("Finished"))
1996 if __name__ == "__main__":