3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
98 return os.path.join(repodir, "icons")
99 return os.path.join(repodir, "icons-%s" % density)
102 def get_icon_dirs(repodir):
103 for density in screen_densities:
104 yield get_icon_dir(repodir, density)
107 def get_all_icon_dirs(repodir):
108 for density in all_screen_densities:
109 yield get_icon_dir(repodir, density)
112 def update_wiki(apps, sortedids, apks):
115 :param apps: fully populated list of all applications
116 :param apks: all apks, except...
118 logging.info("Updating wiki")
120 wikiredircat = 'App Redirects'
122 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
123 path=config['wiki_path'])
124 site.login(config['wiki_user'], config['wiki_password'])
126 generated_redirects = {}
128 for appid in sortedids:
129 app = metadata.App(apps[appid])
133 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
135 for af in sorted(app.AntiFeatures):
136 wikidata += '{{AntiFeature|' + af + '}}\n'
141 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
144 app.added.strftime('%Y-%m-%d') if app.added else '',
145 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
160 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
162 wikidata += app.Summary
163 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
165 wikidata += "=Description=\n"
166 wikidata += metadata.description_wiki(app.Description) + "\n"
168 wikidata += "=Maintainer Notes=\n"
169 if app.MaintainerNotes:
170 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
171 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
173 # Get a list of all packages for this application...
175 gotcurrentver = False
179 if apk['packageName'] == appid:
180 if str(apk['versionCode']) == app.CurrentVersionCode:
183 # Include ones we can't build, as a special case...
184 for build in app.builds:
186 if build.versionCode == app.CurrentVersionCode:
188 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
189 apklist.append({'versionCode': int(build.versionCode),
190 'versionName': build.versionName,
191 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
196 if apk['versionCode'] == int(build.versionCode):
201 apklist.append({'versionCode': int(build.versionCode),
202 'versionName': build.versionName,
203 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
205 if app.CurrentVersionCode == '0':
207 # Sort with most recent first...
208 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
210 wikidata += "=Versions=\n"
211 if len(apklist) == 0:
212 wikidata += "We currently have no versions of this app available."
213 elif not gotcurrentver:
214 wikidata += "We don't have the current version of this app."
216 wikidata += "We have the current version of this app."
217 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
218 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
219 if len(app.NoSourceSince) > 0:
220 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
221 if len(app.CurrentVersion) > 0:
222 wikidata += "The current (recommended) version is " + app.CurrentVersion
223 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
226 wikidata += "==" + apk['versionName'] + "==\n"
228 if 'buildproblem' in apk:
229 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
232 wikidata += "This version is built and signed by "
234 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
236 wikidata += "the original developer.\n\n"
237 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
239 wikidata += '\n[[Category:' + wikicat + ']]\n'
240 if len(app.NoSourceSince) > 0:
241 wikidata += '\n[[Category:Apps missing source code]]\n'
242 if validapks == 0 and not app.Disabled:
243 wikidata += '\n[[Category:Apps with no packages]]\n'
244 if cantupdate and not app.Disabled:
245 wikidata += "\n[[Category:Apps we cannot update]]\n"
246 if buildfails and not app.Disabled:
247 wikidata += "\n[[Category:Apps with failing builds]]\n"
248 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
249 wikidata += '\n[[Category:Apps to Update]]\n'
251 wikidata += '\n[[Category:Apps that are disabled]]\n'
252 if app.UpdateCheckMode == 'None' and not app.Disabled:
253 wikidata += '\n[[Category:Apps with no update check]]\n'
254 for appcat in app.Categories:
255 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
257 # We can't have underscores in the page name, even if they're in
258 # the package ID, because MediaWiki messes with them...
259 pagename = appid.replace('_', ' ')
261 # Drop a trailing newline, because mediawiki is going to drop it anyway
262 # and it we don't we'll think the page has changed when it hasn't...
263 if wikidata.endswith('\n'):
264 wikidata = wikidata[:-1]
266 generated_pages[pagename] = wikidata
268 # Make a redirect from the name to the ID too, unless there's
269 # already an existing page with the name and it isn't a redirect.
271 apppagename = app.Name.replace('_', ' ')
272 apppagename = apppagename.replace('{', '')
273 apppagename = apppagename.replace('}', ' ')
274 apppagename = apppagename.replace(':', ' ')
275 apppagename = apppagename.replace('[', ' ')
276 apppagename = apppagename.replace(']', ' ')
277 # Drop double spaces caused mostly by replacing ':' above
278 apppagename = apppagename.replace(' ', ' ')
279 for expagename in site.allpages(prefix=apppagename,
280 filterredir='nonredirects',
282 if expagename == apppagename:
284 # Another reason not to make the redirect page is if the app name
285 # is the same as it's ID, because that will overwrite the real page
286 # with an redirect to itself! (Although it seems like an odd
287 # scenario this happens a lot, e.g. where there is metadata but no
288 # builds or binaries to extract a name from.
289 if apppagename == pagename:
292 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
294 for tcat, genp in [(wikicat, generated_pages),
295 (wikiredircat, generated_redirects)]:
296 catpages = site.Pages['Category:' + tcat]
298 for page in catpages:
299 existingpages.append(page.name)
300 if page.name in genp:
301 pagetxt = page.edit()
302 if pagetxt != genp[page.name]:
303 logging.debug("Updating modified page " + page.name)
304 page.save(genp[page.name], summary='Auto-updated')
306 logging.debug("Page " + page.name + " is unchanged")
308 logging.warn("Deleting page " + page.name)
309 page.delete('No longer published')
310 for pagename, text in genp.items():
311 logging.debug("Checking " + pagename)
312 if pagename not in existingpages:
313 logging.debug("Creating page " + pagename)
315 newpage = site.Pages[pagename]
316 newpage.save(text, summary='Auto-created')
317 except Exception as e:
318 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
320 # Purge server cache to ensure counts are up to date
321 site.pages['Repository Maintenance'].purge()
324 def delete_disabled_builds(apps, apkcache, repodirs):
325 """Delete disabled build outputs.
327 :param apps: list of all applications, as per metadata.read_metadata
328 :param apkcache: current apk cache information
329 :param repodirs: the repo directories to process
331 for appid, app in apps.items():
332 for build in app['builds']:
333 if not build.disable:
335 apkfilename = common.get_release_filename(app, build)
336 iconfilename = "%s.%s.png" % (
339 for repodir in repodirs:
341 os.path.join(repodir, apkfilename),
342 os.path.join(repodir, apkfilename + '.asc'),
343 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
345 for density in all_screen_densities:
346 repo_dir = get_icon_dir(repodir, density)
347 files.append(os.path.join(repo_dir, iconfilename))
350 if os.path.exists(f):
351 logging.info("Deleting disabled build output " + f)
353 if apkfilename in apkcache:
354 del apkcache[apkfilename]
357 def resize_icon(iconpath, density):
359 if not os.path.isfile(iconpath):
364 fp = open(iconpath, 'rb')
366 size = dpi_to_px(density)
368 if any(length > size for length in im.size):
370 im.thumbnail((size, size), Image.ANTIALIAS)
371 logging.debug("%s was too large at %s - new size is %s" % (
372 iconpath, oldsize, im.size))
373 im.save(iconpath, "PNG")
375 except Exception as e:
376 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
383 def resize_all_icons(repodirs):
384 """Resize all icons that exceed the max size
386 :param repodirs: the repo directories to process
388 for repodir in repodirs:
389 for density in screen_densities:
390 icon_dir = get_icon_dir(repodir, density)
391 icon_glob = os.path.join(icon_dir, '*.png')
392 for iconpath in glob.glob(icon_glob):
393 resize_icon(iconpath, density)
397 """ Get the signing certificate of an apk. To get the same md5 has that
398 Android gets, we encode the .RSA certificate in a specific format and pass
399 it hex-encoded to the md5 digest algorithm.
401 :param apkpath: path to the apk
402 :returns: A string containing the md5 of the signature of the apk or None
403 if an error occurred.
406 with zipfile.ZipFile(apkpath, 'r') as apk:
407 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
410 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
413 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
416 cert = apk.read(certs[0])
418 cert_encoded = common.get_certificate(cert)
420 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
423 def get_cache_file():
424 return os.path.join('tmp', 'apkcache')
428 """Get the cached dict of the APK index
430 Gather information about all the apk files in the repo directory,
431 using cached data if possible. Some of the index operations take a
432 long time, like calculating the SHA-256 and verifying the APK
435 The cache is invalidated if the metadata version is different, or
436 the 'allow_disabled_algorithms' config/option is different. In
437 those cases, there is no easy way to know what has changed from
438 the cache, so just rerun the whole thing.
443 apkcachefile = get_cache_file()
444 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
445 if not options.clean and os.path.exists(apkcachefile):
446 with open(apkcachefile, 'rb') as cf:
447 apkcache = pickle.load(cf, encoding='utf-8')
448 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
449 or apkcache.get('allow_disabled_algorithms') != ada:
454 apkcache["METADATA_VERSION"] = METADATA_VERSION
455 apkcache['allow_disabled_algorithms'] = ada
460 def write_cache(apkcache):
461 apkcachefile = get_cache_file()
462 cache_path = os.path.dirname(apkcachefile)
463 if not os.path.exists(cache_path):
464 os.makedirs(cache_path)
465 with open(apkcachefile, 'wb') as cf:
466 pickle.dump(apkcache, cf)
469 def get_icon_bytes(apkzip, iconsrc):
470 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
472 return apkzip.read(iconsrc)
474 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
477 def sha256sum(filename):
478 '''Calculate the sha256 of the given file'''
479 sha = hashlib.sha256()
480 with open(filename, 'rb') as f:
486 return sha.hexdigest()
489 def has_known_vulnerability(filename):
490 """checks for known vulnerabilities in the APK
492 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
493 version. Google also enforces this:
494 https://support.google.com/faqs/answer/6376725?hl=en
496 Checks whether there are more than one classes.dex or AndroidManifest.xml
497 files, which is invalid and an essential part of the "Master Key" attack.
499 http://www.saurik.com/id/17
502 # statically load this pattern
503 if not hasattr(has_known_vulnerability, "pattern"):
504 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
507 with zipfile.ZipFile(filename) as zf:
508 for name in zf.namelist():
509 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
512 chunk = lib.read(4096)
515 m = has_known_vulnerability.pattern.search(chunk)
517 version = m.group(1).decode('ascii')
518 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
519 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
520 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
521 logging.debug(_('"{path}" contains recent {name} ({version})')
522 .format(path=filename, name=name, version=version))
524 logging.warning(_('"{path}" contains outdated {name} ({version})')
525 .format(path=filename, name=name, version=version))
528 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
529 if name in files_in_apk:
531 files_in_apk.add(name)
536 def insert_obbs(repodir, apps, apks):
537 """Scans the .obb files in a given repo directory and adds them to the
538 relevant APK instances. OBB files have versionCodes like APK
539 files, and they are loosely associated. If there is an OBB file
540 present, then any APK with the same or higher versionCode will use
541 that OBB file. There are two OBB types: main and patch, each APK
542 can only have only have one of each.
544 https://developer.android.com/google/play/expansion-files.html
546 :param repodir: repo directory to scan
547 :param apps: list of current, valid apps
548 :param apks: current information on all APKs
552 def obbWarnDelete(f, msg):
553 logging.warning(msg + ' ' + f)
554 if options.delete_unknown:
555 logging.error(_("Deleting unknown file: {path}").format(path=f))
559 java_Integer_MIN_VALUE = -pow(2, 31)
560 currentPackageNames = apps.keys()
561 for f in glob.glob(os.path.join(repodir, '*.obb')):
562 obbfile = os.path.basename(f)
563 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
564 chunks = obbfile.split('.')
565 if chunks[0] != 'main' and chunks[0] != 'patch':
566 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
568 if not re.match(r'^-?[0-9]+$', chunks[1]):
569 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
570 .format(name=chunks[0]))
572 versionCode = int(chunks[1])
573 packagename = ".".join(chunks[2:-1])
575 highestVersionCode = java_Integer_MIN_VALUE
576 if packagename not in currentPackageNames:
577 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
580 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
581 highestVersionCode = apk['versionCode']
582 if versionCode > highestVersionCode:
583 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
584 .format(integer=str(versionCode)))
586 obbsha256 = sha256sum(f)
587 obbs.append((packagename, versionCode, obbfile, obbsha256))
590 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
591 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
592 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
593 apk['obbMainFile'] = obbfile
594 apk['obbMainFileSha256'] = obbsha256
595 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
596 apk['obbPatchFile'] = obbfile
597 apk['obbPatchFileSha256'] = obbsha256
598 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
602 def translate_per_build_anti_features(apps, apks):
603 """Grab the anti-features list from the build metadata
605 For most Anti-Features, they are really most applicable per-APK,
606 not for an app. An app can fix a vulnerability, add/remove
607 tracking, etc. This reads the 'antifeatures' list from the Build
608 entries in the fdroiddata metadata file, then transforms it into
609 the 'antiFeatures' list of unique items for the index.
611 The field key is all lower case in the metadata file to match the
612 rest of the Build fields. It is 'antiFeatures' camel case in the
613 implementation, index, and fdroidclient since it is translated
614 from the build 'antifeatures' field, not directly included.
618 antiFeatures = dict()
619 for packageName, app in apps.items():
621 for build in app['builds']:
622 afl = build.get('antifeatures')
624 d[int(build.versionCode)] = afl
626 antiFeatures[packageName] = d
629 d = antiFeatures.get(apk['packageName'])
631 afl = d.get(apk['versionCode'])
633 apk['antiFeatures'].update(afl)
636 def _get_localized_dict(app, locale):
637 '''get the dict to add localized store metadata to'''
638 if 'localized' not in app:
639 app['localized'] = collections.OrderedDict()
640 if locale not in app['localized']:
641 app['localized'][locale] = collections.OrderedDict()
642 return app['localized'][locale]
645 def _set_localized_text_entry(app, locale, key, f):
646 limit = config['char_limits'][key]
647 localized = _get_localized_dict(app, locale)
649 text = fp.read()[:limit]
651 localized[key] = text
654 def _set_author_entry(app, key, f):
655 limit = config['char_limits']['author']
657 text = fp.read()[:limit]
662 def copy_triple_t_store_metadata(apps):
663 """Include store metadata from the app's source repo
665 The Triple-T Gradle Play Publisher is a plugin that has a standard
666 file layout for all of the metadata and graphics that the Google
667 Play Store accepts. Since F-Droid has the git repo, it can just
668 pluck those files directly. This method reads any text files into
669 the app dict, then copies any graphics into the fdroid repo
672 This needs to be run before insert_localized_app_metadata() so that
673 the graphics files that are copied into the fdroid repo get
676 https://github.com/Triple-T/gradle-play-publisher#upload-images
677 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
681 if not os.path.isdir('build'):
682 return # nothing to do
684 for packageName, app in apps.items():
685 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
686 logging.debug('Triple-T Gradle Play Publisher: ' + d)
687 for root, dirs, files in os.walk(d):
688 segments = root.split('/')
689 locale = segments[-2]
691 if f == 'fulldescription':
692 _set_localized_text_entry(app, locale, 'description',
693 os.path.join(root, f))
695 elif f == 'shortdescription':
696 _set_localized_text_entry(app, locale, 'summary',
697 os.path.join(root, f))
700 _set_localized_text_entry(app, locale, 'name',
701 os.path.join(root, f))
704 _set_localized_text_entry(app, locale, 'video',
705 os.path.join(root, f))
707 elif f == 'whatsnew':
708 _set_localized_text_entry(app, segments[-1], 'whatsNew',
709 os.path.join(root, f))
711 elif f == 'contactEmail':
712 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
714 elif f == 'contactPhone':
715 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
717 elif f == 'contactWebsite':
718 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
721 base, extension = common.get_extension(f)
722 dirname = os.path.basename(root)
723 if extension in ALLOWED_EXTENSIONS \
724 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
725 if segments[-2] == 'listing':
726 locale = segments[-3]
728 locale = segments[-2]
729 destdir = os.path.join('repo', packageName, locale, dirname)
730 os.makedirs(destdir, mode=0o755, exist_ok=True)
731 sourcefile = os.path.join(root, f)
732 destfile = os.path.join(destdir, os.path.basename(f))
733 logging.debug('copying ' + sourcefile + ' ' + destfile)
734 shutil.copy(sourcefile, destfile)
737 def insert_localized_app_metadata(apps):
738 """scans standard locations for graphics and localized text
740 Scans for localized description files, store graphics, and
741 screenshot PNG files in statically defined screenshots directory
742 and adds them to the app metadata. The screenshots and graphic
743 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
744 and must be in the following layout:
745 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
747 repo/packageName/locale/featureGraphic.png
748 repo/packageName/locale/phoneScreenshots/1.png
749 repo/packageName/locale/phoneScreenshots/2.png
751 The changelog files must be text files named with the versionCode
752 ending with ".txt" and must be in the following layout:
753 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
755 repo/packageName/locale/changelogs/12345.txt
757 This will scan the each app's source repo then the metadata/ dir
758 for these standard locations of changelog files. If it finds
759 them, they will be added to the dict of all packages, with the
760 versions in the metadata/ folder taking precendence over the what
761 is in the app's source repo.
763 Where "packageName" is the app's packageName and "locale" is the locale
764 of the graphics, e.g. what language they are in, using the IETF RFC5646
765 format (en-US, fr-CA, es-MX, etc).
767 This will also scan the app's git for a fastlane folder, and the
768 metadata/ folder and the apps' source repos for standard locations
769 of graphic and screenshot files. If it finds them, it will copy
770 them into the repo. The fastlane files follow this pattern:
771 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
775 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
776 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
777 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
778 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
780 for srcd in sorted(sourcedirs):
781 if not os.path.isdir(srcd):
783 for root, dirs, files in os.walk(srcd):
784 segments = root.split('/')
785 packageName = segments[1]
786 if packageName not in apps:
787 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
789 locale = segments[-1]
790 destdir = os.path.join('repo', packageName, locale)
792 # flavours specified in build receipt
794 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
795 and 'gradle' in apps[packageName].builds[-1]:
796 build_flavours = apps[packageName].builds[-1].gradle
798 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
799 logging.debug("ignoring due to wrong flavour")
803 if f in ('description.txt', 'full_description.txt'):
804 _set_localized_text_entry(apps[packageName], locale, 'description',
805 os.path.join(root, f))
807 elif f in ('summary.txt', 'short_description.txt'):
808 _set_localized_text_entry(apps[packageName], locale, 'summary',
809 os.path.join(root, f))
811 elif f in ('name.txt', 'title.txt'):
812 _set_localized_text_entry(apps[packageName], locale, 'name',
813 os.path.join(root, f))
815 elif f == 'video.txt':
816 _set_localized_text_entry(apps[packageName], locale, 'video',
817 os.path.join(root, f))
819 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
820 locale = segments[-2]
821 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
822 os.path.join(root, f))
825 base, extension = common.get_extension(f)
826 if locale == 'images':
827 locale = segments[-2]
828 destdir = os.path.join('repo', packageName, locale)
829 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
830 os.makedirs(destdir, mode=0o755, exist_ok=True)
831 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
832 shutil.copy(os.path.join(root, f), destdir)
834 if d in SCREENSHOT_DIRS:
835 if locale == 'images':
836 locale = segments[-2]
837 destdir = os.path.join('repo', packageName, locale)
838 for f in glob.glob(os.path.join(root, d, '*.*')):
839 _ignored, extension = common.get_extension(f)
840 if extension in ALLOWED_EXTENSIONS:
841 screenshotdestdir = os.path.join(destdir, d)
842 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
843 logging.debug('copying ' + f + ' ' + screenshotdestdir)
844 shutil.copy(f, screenshotdestdir)
846 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
848 if not os.path.isdir(d):
850 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
851 if not os.path.isfile(f):
853 segments = f.split('/')
854 packageName = segments[1]
856 screenshotdir = segments[3]
857 filename = os.path.basename(f)
858 base, extension = common.get_extension(filename)
860 if packageName not in apps:
861 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
862 .format(path=filename, name=packageName))
864 graphics = _get_localized_dict(apps[packageName], locale)
866 if extension not in ALLOWED_EXTENSIONS:
867 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
868 elif base in GRAPHIC_NAMES:
869 # there can only be zero or one of these per locale
870 graphics[base] = filename
871 elif screenshotdir in SCREENSHOT_DIRS:
872 # there can any number of these per locale
873 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
874 if screenshotdir not in graphics:
875 graphics[screenshotdir] = []
876 graphics[screenshotdir].append(filename)
878 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
881 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
882 """Scan a repo for all files with an extension except APK/OBB
884 :param apkcache: current cached info about all repo files
885 :param repodir: repo directory to scan
886 :param knownapks: list of all known files, as per metadata.read_metadata
887 :param use_date_from_file: use date from file (instead of current date)
888 for newly added files
893 repodir = repodir.encode('utf-8')
894 for name in os.listdir(repodir):
895 file_extension = common.get_file_extension(name)
896 if file_extension == 'apk' or file_extension == 'obb':
898 filename = os.path.join(repodir, name)
899 name_utf8 = name.decode('utf-8')
900 if filename.endswith(b'_src.tar.gz'):
901 logging.debug(_('skipping source tarball: {path}')
902 .format(path=filename.decode('utf-8')))
904 if not common.is_repo_file(filename):
906 stat = os.stat(filename)
907 if stat.st_size == 0:
908 raise FDroidException(_('{path} is zero size!')
909 .format(path=filename))
911 shasum = sha256sum(filename)
914 repo_file = apkcache[name]
915 # added time is cached as tuple but used here as datetime instance
916 if 'added' in repo_file:
917 a = repo_file['added']
918 if isinstance(a, datetime):
919 repo_file['added'] = a
921 repo_file['added'] = datetime(*a[:6])
922 if repo_file.get('hash') == shasum:
923 logging.debug(_("Reading {apkfilename} from cache")
924 .format(apkfilename=name_utf8))
927 logging.debug(_("Ignoring stale cache data for {apkfilename}")
928 .format(apkfilename=name_utf8))
931 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
932 repo_file = collections.OrderedDict()
933 repo_file['name'] = os.path.splitext(name_utf8)[0]
934 # TODO rename apkname globally to something more generic
935 repo_file['apkName'] = name_utf8
936 repo_file['hash'] = shasum
937 repo_file['hashType'] = 'sha256'
938 repo_file['versionCode'] = 0
939 repo_file['versionName'] = shasum
940 # the static ID is the SHA256 unless it is set in the metadata
941 repo_file['packageName'] = shasum
943 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
945 repo_file['packageName'] = m.group(1)
946 repo_file['versionCode'] = int(m.group(2))
947 srcfilename = name + b'_src.tar.gz'
948 if os.path.exists(os.path.join(repodir, srcfilename)):
949 repo_file['srcname'] = srcfilename.decode('utf-8')
950 repo_file['size'] = stat.st_size
952 apkcache[name] = repo_file
955 if use_date_from_file:
956 timestamp = stat.st_ctime
957 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
959 default_date_param = None
961 # Record in knownapks, getting the added date at the same time..
962 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
963 default_date=default_date_param)
965 repo_file['added'] = added
967 repo_files.append(repo_file)
969 return repo_files, cachechanged
972 def scan_apk(apk_file):
974 Scans an APK file and returns dictionary with metadata of the APK.
976 Attention: This does *not* verify that the APK signature is correct.
978 :param apk_file: The (ideally absolute) path to the APK file
979 :raises BuildException
980 :return A dict containing APK metadata
983 'hash': sha256sum(apk_file),
984 'hashType': 'sha256',
985 'uses-permission': [],
986 'uses-permission-sdk-23': [],
990 'antiFeatures': set(),
993 if SdkToolsPopen(['aapt', 'version'], output=False):
994 scan_apk_aapt(apk, apk_file)
996 scan_apk_androguard(apk, apk_file)
998 # Get the signature, or rather the signing key fingerprints
999 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1000 apk['sig'] = getsig(apk_file)
1002 raise BuildException("Failed to get apk signature")
1003 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1005 if not apk.get('signer'):
1006 raise BuildException("Failed to get apk signing key fingerprint")
1008 # Get size of the APK
1009 apk['size'] = os.path.getsize(apk_file)
1011 if 'minSdkVersion' not in apk:
1012 logging.warning("No SDK version information found in {0}".format(apk_file))
1013 apk['minSdkVersion'] = 1
1015 # Check for known vulnerabilities
1016 if has_known_vulnerability(apk_file):
1017 apk['antiFeatures'].add('KnownVuln')
1022 def scan_apk_aapt(apk, apkfile):
1023 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1024 if p.returncode != 0:
1025 if options.delete_unknown:
1026 if os.path.exists(apkfile):
1027 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1030 logging.error("Could not find {0} to remove it".format(apkfile))
1032 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1033 raise BuildException(_("Invalid APK"))
1034 for line in p.output.splitlines():
1035 if line.startswith("package:"):
1037 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1038 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1039 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1040 except Exception as e:
1041 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1042 elif line.startswith("application:"):
1043 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1044 # Keep path to non-dpi icon in case we need it
1045 match = re.match(APK_ICON_PAT_NODPI, line)
1047 apk['icons_src']['-1'] = match.group(1)
1048 elif line.startswith("launchable-activity:"):
1049 # Only use launchable-activity as fallback to application
1051 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1052 if '-1' not in apk['icons_src']:
1053 match = re.match(APK_ICON_PAT_NODPI, line)
1055 apk['icons_src']['-1'] = match.group(1)
1056 elif line.startswith("application-icon-"):
1057 match = re.match(APK_ICON_PAT, line)
1059 density = match.group(1)
1060 path = match.group(2)
1061 apk['icons_src'][density] = path
1062 elif line.startswith("sdkVersion:"):
1063 m = re.match(APK_SDK_VERSION_PAT, line)
1065 logging.error(line.replace('sdkVersion:', '')
1066 + ' is not a valid minSdkVersion!')
1068 apk['minSdkVersion'] = m.group(1)
1069 # if target not set, default to min
1070 if 'targetSdkVersion' not in apk:
1071 apk['targetSdkVersion'] = m.group(1)
1072 elif line.startswith("targetSdkVersion:"):
1073 m = re.match(APK_SDK_VERSION_PAT, line)
1075 logging.error(line.replace('targetSdkVersion:', '')
1076 + ' is not a valid targetSdkVersion!')
1078 apk['targetSdkVersion'] = m.group(1)
1079 elif line.startswith("maxSdkVersion:"):
1080 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1081 elif line.startswith("native-code:"):
1082 apk['nativecode'] = []
1083 for arch in line[13:].split(' '):
1084 apk['nativecode'].append(arch[1:-1])
1085 elif line.startswith('uses-permission:'):
1086 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1087 if perm_match['maxSdkVersion']:
1088 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1089 permission = UsesPermission(
1091 perm_match['maxSdkVersion']
1094 apk['uses-permission'].append(permission)
1095 elif line.startswith('uses-permission-sdk-23:'):
1096 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1097 if perm_match['maxSdkVersion']:
1098 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1099 permission_sdk_23 = UsesPermissionSdk23(
1101 perm_match['maxSdkVersion']
1104 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1106 elif line.startswith('uses-feature:'):
1107 feature = re.match(APK_FEATURE_PAT, line).group(1)
1108 # Filter out this, it's only added with the latest SDK tools and
1109 # causes problems for lots of apps.
1110 if feature != "android.hardware.screen.portrait" \
1111 and feature != "android.hardware.screen.landscape":
1112 if feature.startswith("android.feature."):
1113 feature = feature[16:]
1114 apk['features'].add(feature)
1117 def scan_apk_androguard(apk, apkfile):
1119 from androguard.core.bytecodes.apk import APK
1120 apkobject = APK(apkfile)
1121 if apkobject.is_valid_APK():
1122 arsc = apkobject.get_android_resources()
1124 if options.delete_unknown:
1125 if os.path.exists(apkfile):
1126 logging.error(_("Failed to get apk information, deleting {path}")
1127 .format(path=apkfile))
1130 logging.error(_("Could not find {path} to remove it")
1131 .format(path=apkfile))
1133 logging.error(_("Failed to get apk information, skipping {path}")
1134 .format(path=apkfile))
1135 raise BuildException(_("Invalid APK"))
1137 raise FDroidException("androguard library is not installed and aapt not present")
1138 except FileNotFoundError:
1139 logging.error(_("Could not open apk file for analysis"))
1140 raise BuildException(_("Invalid APK"))
1142 apk['packageName'] = apkobject.get_package()
1143 apk['versionCode'] = int(apkobject.get_androidversion_code())
1144 apk['versionName'] = apkobject.get_androidversion_name()
1145 if apk['versionName'][0] == "@":
1146 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1147 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1148 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1149 apk['name'] = apkobject.get_app_name()
1151 if apkobject.get_max_sdk_version() is not None:
1152 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1153 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1154 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1156 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1157 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1159 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1161 for file in apkobject.get_files():
1162 d_re = density_re.match(file)
1164 folder = d_re.group(1).split('-')
1166 resolution = folder[1]
1169 density = screen_resolutions[resolution]
1170 apk['icons_src'][density] = d_re.group(0)
1172 if apk['icons_src'].get('-1') is None:
1173 apk['icons_src']['-1'] = apk['icons_src']['160']
1175 arch_re = re.compile("^lib/(.*)/.*$")
1176 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1178 apk['nativecode'] = []
1179 apk['nativecode'].extend(sorted(list(arch)))
1181 xml = apkobject.get_android_manifest_xml()
1183 for item in xml.getElementsByTagName('uses-permission'):
1184 name = str(item.getAttribute("android:name"))
1185 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1186 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1187 permission = UsesPermission(
1191 apk['uses-permission'].append(permission)
1193 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1194 name = str(item.getAttribute("android:name"))
1195 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1196 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1197 permission_sdk_23 = UsesPermissionSdk23(
1201 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1203 for item in xml.getElementsByTagName('uses-feature'):
1204 feature = str(item.getAttribute("android:name"))
1205 if feature != "android.hardware.screen.portrait" \
1206 and feature != "android.hardware.screen.landscape":
1207 if feature.startswith("android.feature."):
1208 feature = feature[16:]
1209 apk['features'].append(feature)
1212 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1213 allow_disabled_algorithms=False, archive_bad_sig=False):
1214 """Processes the apk with the given filename in the given repo directory.
1216 This also extracts the icons.
1218 :param apkcache: current apk cache information
1219 :param apkfilename: the filename of the apk to scan
1220 :param repodir: repo directory to scan
1221 :param knownapks: known apks info
1222 :param use_date_from_apk: use date from APK (instead of current date)
1223 for newly added APKs
1224 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1225 disabled algorithms in the signature (e.g. MD5)
1226 :param archive_bad_sig: move APKs with a bad signature to the archive
1227 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1228 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1232 apkfile = os.path.join(repodir, apkfilename)
1234 cachechanged = False
1236 if apkfilename in apkcache:
1237 apk = apkcache[apkfilename]
1238 if apk.get('hash') == sha256sum(apkfile):
1239 logging.debug(_("Reading {apkfilename} from cache")
1240 .format(apkfilename=apkfilename))
1243 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1244 .format(apkfilename=apkfilename))
1247 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1250 apk = scan_apk(apkfile)
1251 except BuildException:
1252 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1253 .format(apkfilename=apkfilename))
1254 return True, None, False
1256 # Check for debuggable apks...
1257 if common.isApkAndDebuggable(apkfile):
1258 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1260 if options.rename_apks:
1261 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1262 std_short_name = os.path.join(repodir, n)
1263 if apkfile != std_short_name:
1264 if os.path.exists(std_short_name):
1265 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1266 if apkfile != std_long_name:
1267 if os.path.exists(std_long_name):
1268 dupdir = os.path.join('duplicates', repodir)
1269 if not os.path.isdir(dupdir):
1270 os.makedirs(dupdir, exist_ok=True)
1271 dupfile = os.path.join('duplicates', std_long_name)
1272 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1273 os.rename(apkfile, dupfile)
1274 return True, None, False
1276 os.rename(apkfile, std_long_name)
1277 apkfile = std_long_name
1279 os.rename(apkfile, std_short_name)
1280 apkfile = std_short_name
1281 apkfilename = apkfile[len(repodir) + 1:]
1283 apk['apkName'] = apkfilename
1284 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1285 if os.path.exists(os.path.join(repodir, srcfilename)):
1286 apk['srcname'] = srcfilename
1288 # verify the jar signature is correct, allow deprecated
1289 # algorithms only if the APK is in the archive.
1291 if not common.verify_apk_signature(apkfile):
1292 if repodir == 'archive' or allow_disabled_algorithms:
1293 if common.verify_old_apk_signature(apkfile):
1294 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1302 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1303 .format(apkfilename=apkfilename))
1304 move_apk_between_sections(repodir, 'archive', apk)
1306 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1307 .format(apkfilename=apkfilename))
1308 return True, None, False
1310 apkzip = zipfile.ZipFile(apkfile, 'r')
1312 manifest = apkzip.getinfo('AndroidManifest.xml')
1313 if manifest.date_time[1] == 0: # month can't be zero
1314 logging.debug(_('AndroidManifest.xml has no date'))
1316 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1318 # extract icons from APK zip file
1319 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1321 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1323 apkzip.close() # ensure that APK zip file gets closed
1325 # resize existing icons for densities missing in the APK
1326 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1328 if use_date_from_apk and manifest.date_time[1] != 0:
1329 default_date_param = datetime(*manifest.date_time)
1331 default_date_param = None
1333 # Record in known apks, getting the added date at the same time..
1334 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1335 default_date=default_date_param)
1337 apk['added'] = added
1339 apkcache[apkfilename] = apk
1342 return False, apk, cachechanged
1345 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1346 """Processes the apks in the given repo directory.
1348 This also extracts the icons.
1350 :param apkcache: current apk cache information
1351 :param repodir: repo directory to scan
1352 :param knownapks: known apks info
1353 :param use_date_from_apk: use date from APK (instead of current date)
1354 for newly added APKs
1355 :returns: (apks, cachechanged) where apks is a list of apk information,
1356 and cachechanged is True if the apkcache got changed.
1359 cachechanged = False
1361 for icon_dir in get_all_icon_dirs(repodir):
1362 if os.path.exists(icon_dir):
1364 shutil.rmtree(icon_dir)
1365 os.makedirs(icon_dir)
1367 os.makedirs(icon_dir)
1370 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1371 apkfilename = apkfile[len(repodir) + 1:]
1372 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1373 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1374 use_date_from_apk, ada, True)
1378 cachechanged = cachechanged or cachethis
1380 return apks, cachechanged
1383 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1385 Extracts icons from the given APK zip in various densities,
1386 saves them into given repo directory
1387 and stores their names in the APK metadata dictionary.
1389 :param icon_filename: A string representing the icon's file name
1390 :param apk: A populated dictionary containing APK metadata.
1391 Needs to have 'icons_src' key
1392 :param apkzip: An opened zipfile.ZipFile of the APK file
1393 :param repo_dir: The directory of the APK's repository
1394 :return: A list of icon densities that are missing
1396 empty_densities = []
1397 for density in screen_densities:
1398 if density not in apk['icons_src']:
1399 empty_densities.append(density)
1401 icon_src = apk['icons_src'][density]
1402 icon_dir = get_icon_dir(repo_dir, density)
1403 icon_dest = os.path.join(icon_dir, icon_filename)
1405 # Extract the icon files per density
1406 if icon_src.endswith('.xml'):
1407 png = os.path.basename(icon_src)[:-4] + '.png'
1408 for f in apkzip.namelist():
1410 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1411 if m and screen_resolutions[m.group(2)] == density:
1413 if icon_src.endswith('.xml'):
1414 empty_densities.append(density)
1417 with open(icon_dest, 'wb') as f:
1418 f.write(get_icon_bytes(apkzip, icon_src))
1419 apk['icons'][density] = icon_filename
1420 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1421 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1422 del apk['icons_src'][density]
1423 empty_densities.append(density)
1425 if '-1' in apk['icons_src']:
1426 icon_src = apk['icons_src']['-1']
1427 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1428 with open(icon_path, 'wb') as f:
1429 f.write(get_icon_bytes(apkzip, icon_src))
1431 im = Image.open(icon_path)
1432 dpi = px_to_dpi(im.size[0])
1433 for density in screen_densities:
1434 if density in apk['icons']:
1436 if density == screen_densities[-1] or dpi >= int(density):
1437 apk['icons'][density] = icon_filename
1438 shutil.move(icon_path,
1439 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1440 empty_densities.remove(density)
1442 except Exception as e:
1443 logging.warning(_("Failed reading {path}: {error}")
1444 .format(path=icon_path, error=e))
1447 apk['icon'] = icon_filename
1449 return empty_densities
1452 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1454 Resize existing icons for densities missing in the APK to ensure all densities are available
1456 :param empty_densities: A list of icon densities that are missing
1457 :param icon_filename: A string representing the icon's file name
1458 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1459 :param repo_dir: The directory of the APK's repository
1461 # First try resizing down to not lose quality
1463 for density in screen_densities:
1464 if density not in empty_densities:
1465 last_density = density
1467 if last_density is None:
1469 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1471 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1472 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1475 fp = open(last_icon_path, 'rb')
1478 size = dpi_to_px(density)
1480 im.thumbnail((size, size), Image.ANTIALIAS)
1481 im.save(icon_path, "PNG")
1482 empty_densities.remove(density)
1483 except Exception as e:
1484 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1489 # Then just copy from the highest resolution available
1491 for density in reversed(screen_densities):
1492 if density not in empty_densities:
1493 last_density = density
1496 if last_density is None:
1500 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1501 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1503 empty_densities.remove(density)
1505 for density in screen_densities:
1506 icon_dir = get_icon_dir(repo_dir, density)
1507 icon_dest = os.path.join(icon_dir, icon_filename)
1508 resize_icon(icon_dest, density)
1510 # Copy from icons-mdpi to icons since mdpi is the baseline density
1511 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1512 if os.path.isfile(baseline):
1513 apk['icons']['0'] = icon_filename
1514 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1517 def apply_info_from_latest_apk(apps, apks):
1519 Some information from the apks needs to be applied up to the application level.
1520 When doing this, we use the info from the most recent version's apk.
1521 We deal with figuring out when the app was added and last updated at the same time.
1523 for appid, app in apps.items():
1524 bestver = UNSET_VERSION_CODE
1526 if apk['packageName'] == appid:
1527 if apk['versionCode'] > bestver:
1528 bestver = apk['versionCode']
1532 if not app.added or apk['added'] < app.added:
1533 app.added = apk['added']
1534 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1535 app.lastUpdated = apk['added']
1538 logging.debug("Don't know when " + appid + " was added")
1539 if not app.lastUpdated:
1540 logging.debug("Don't know when " + appid + " was last updated")
1542 if bestver == UNSET_VERSION_CODE:
1544 if app.Name is None:
1545 app.Name = app.AutoName or appid
1547 logging.debug("Application " + appid + " has no packages")
1549 if app.Name is None:
1550 app.Name = bestapk['name']
1551 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1552 if app.CurrentVersionCode is None:
1553 app.CurrentVersionCode = str(bestver)
1556 def make_categories_txt(repodir, categories):
1557 '''Write a category list in the repo to allow quick access'''
1559 for cat in sorted(categories):
1560 catdata += cat + '\n'
1561 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1565 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1567 def filter_apk_list_sorted(apk_list):
1569 for apk in apk_list:
1570 if apk['packageName'] == appid:
1573 # Sort the apk list by version code. First is highest/newest.
1574 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1576 for appid, app in apps.items():
1578 if app.ArchivePolicy:
1579 keepversions = int(app.ArchivePolicy[:-9])
1581 keepversions = defaultkeepversions
1583 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1584 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1586 current_app_apks = filter_apk_list_sorted(apks)
1587 if len(current_app_apks) > keepversions:
1588 # Move back the ones we don't want.
1589 for apk in current_app_apks[keepversions:]:
1590 move_apk_between_sections(repodir, archivedir, apk)
1591 archapks.append(apk)
1594 current_app_archapks = filter_apk_list_sorted(archapks)
1595 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1597 # Move forward the ones we want again, except DisableAlgorithm
1598 for apk in current_app_archapks:
1599 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1600 move_apk_between_sections(archivedir, repodir, apk)
1601 archapks.remove(apk)
1604 if kept == keepversions:
1608 def move_apk_between_sections(from_dir, to_dir, apk):
1609 """move an APK from repo to archive or vice versa"""
1611 def _move_file(from_dir, to_dir, filename, ignore_missing):
1612 from_path = os.path.join(from_dir, filename)
1613 if ignore_missing and not os.path.exists(from_path):
1615 to_path = os.path.join(to_dir, filename)
1616 if not os.path.exists(to_dir):
1618 shutil.move(from_path, to_path)
1620 if from_dir == to_dir:
1623 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1624 _move_file(from_dir, to_dir, apk['apkName'], False)
1625 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1626 for density in all_screen_densities:
1627 from_icon_dir = get_icon_dir(from_dir, density)
1628 to_icon_dir = get_icon_dir(to_dir, density)
1629 if density not in apk.get('icons', []):
1631 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1632 if 'srcname' in apk:
1633 _move_file(from_dir, to_dir, apk['srcname'], False)
1636 def add_apks_to_per_app_repos(repodir, apks):
1637 apks_per_app = dict()
1639 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1640 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1641 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1642 apks_per_app[apk['packageName']] = apk
1644 if not os.path.exists(apk['per_app_icons']):
1645 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1646 os.makedirs(apk['per_app_icons'])
1648 apkpath = os.path.join(repodir, apk['apkName'])
1649 shutil.copy(apkpath, apk['per_app_repo'])
1650 apksigpath = apkpath + '.sig'
1651 if os.path.exists(apksigpath):
1652 shutil.copy(apksigpath, apk['per_app_repo'])
1653 apkascpath = apkpath + '.asc'
1654 if os.path.exists(apkascpath):
1655 shutil.copy(apkascpath, apk['per_app_repo'])
1658 def create_metadata_from_template(apk):
1659 '''create a new metadata file using internal or external template
1661 Generate warnings for apk's with no metadata (or create skeleton
1662 metadata files, if requested on the command line). Though the
1663 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1664 since those impose things on the metadata file made from the
1665 template: field sort order, empty field value, formatting, etc.
1669 if os.path.exists('template.yml'):
1670 with open('template.yml') as f:
1672 if 'name' in apk and apk['name'] != '':
1673 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1674 r'\1 ' + apk['name'],
1676 flags=re.IGNORECASE | re.MULTILINE)
1678 logging.warning(_('{appid} does not have a name! Using package name instead.')
1679 .format(appid=apk['packageName']))
1680 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1681 r'\1 ' + apk['packageName'],
1683 flags=re.IGNORECASE | re.MULTILINE)
1684 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1688 app['Categories'] = [os.path.basename(os.getcwd())]
1689 # include some blanks as part of the template
1690 app['AuthorName'] = ''
1693 app['IssueTracker'] = ''
1694 app['SourceCode'] = ''
1695 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1696 if 'name' in apk and apk['name'] != '':
1697 app['Name'] = apk['name']
1699 logging.warning(_('{appid} does not have a name! Using package name instead.')
1700 .format(appid=apk['packageName']))
1701 app['Name'] = apk['packageName']
1702 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1703 yaml.dump(app, f, default_flow_style=False)
1704 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1713 global config, options
1715 # Parse command line...
1716 parser = ArgumentParser()
1717 common.setup_global_opts(parser)
1718 parser.add_argument("--create-key", action="store_true", default=False,
1719 help=_("Add a repo signing key to an unsigned repo"))
1720 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1721 help=_("Add skeleton metadata files for APKs that are missing them"))
1722 parser.add_argument("--delete-unknown", action="store_true", default=False,
1723 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1724 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1725 help=_("Report on build data status"))
1726 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1727 help=_("Interactively ask about things that need updating."))
1728 parser.add_argument("-I", "--icons", action="store_true", default=False,
1729 help=_("Resize all the icons exceeding the max pixel size and exit"))
1730 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1731 help=_("Specify editor to use in interactive mode. Default " +
1732 "is {path}").format(path='/etc/alternatives/editor'))
1733 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1734 help=_("Update the wiki"))
1735 parser.add_argument("--pretty", action="store_true", default=False,
1736 help=_("Produce human-readable XML/JSON for index files"))
1737 parser.add_argument("--clean", action="store_true", default=False,
1738 help=_("Clean update - don't uses caches, reprocess all APKs"))
1739 parser.add_argument("--nosign", action="store_true", default=False,
1740 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1741 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1742 help=_("Use date from APK instead of current time for newly added APKs"))
1743 parser.add_argument("--rename-apks", action="store_true", default=False,
1744 help=_("Rename APK files that do not match package.name_123.apk"))
1745 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1746 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1747 metadata.add_metadata_arguments(parser)
1748 options = parser.parse_args()
1749 metadata.warnings_action = options.W
1751 config = common.read_config(options)
1753 if not ('jarsigner' in config and 'keytool' in config):
1754 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1757 if config['archive_older'] != 0:
1758 repodirs.append('archive')
1759 if not os.path.exists('archive'):
1763 resize_all_icons(repodirs)
1766 if options.rename_apks:
1767 options.clean = True
1769 # check that icons exist now, rather than fail at the end of `fdroid update`
1770 for k in ['repo_icon', 'archive_icon']:
1772 if not os.path.exists(config[k]):
1773 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1774 .format(name=k, path=config[k]))
1777 # if the user asks to create a keystore, do it now, reusing whatever it can
1778 if options.create_key:
1779 if os.path.exists(config['keystore']):
1780 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1781 logging.critical("\t'" + config['keystore'] + "'")
1784 if 'repo_keyalias' not in config:
1785 config['repo_keyalias'] = socket.getfqdn()
1786 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1787 if 'keydname' not in config:
1788 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1789 common.write_to_config(config, 'keydname', config['keydname'])
1790 if 'keystore' not in config:
1791 config['keystore'] = common.default_config['keystore']
1792 common.write_to_config(config, 'keystore', config['keystore'])
1794 password = common.genpassword()
1795 if 'keystorepass' not in config:
1796 config['keystorepass'] = password
1797 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1798 if 'keypass' not in config:
1799 config['keypass'] = password
1800 common.write_to_config(config, 'keypass', config['keypass'])
1801 common.genkeystore(config)
1804 apps = metadata.read_metadata()
1806 # Generate a list of categories...
1808 for app in apps.values():
1809 categories.update(app.Categories)
1811 # Read known apks data (will be updated and written back when we've finished)
1812 knownapks = common.KnownApks()
1815 apkcache = get_cache()
1817 # Delete builds for disabled apps
1818 delete_disabled_builds(apps, apkcache, repodirs)
1820 # Scan all apks in the main repo
1821 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1823 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1824 options.use_date_from_apk)
1825 cachechanged = cachechanged or fcachechanged
1828 if apk['packageName'] not in apps:
1829 if options.create_metadata:
1830 create_metadata_from_template(apk)
1831 apps = metadata.read_metadata()
1833 msg = _("{apkfilename} ({appid}) has no metadata!") \
1834 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1835 if options.delete_unknown:
1836 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1837 .format(apkfilename=apk['apkName']))
1838 rmf = os.path.join(repodirs[0], apk['apkName'])
1839 if not os.path.exists(rmf):
1840 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1844 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1846 copy_triple_t_store_metadata(apps)
1847 insert_obbs(repodirs[0], apps, apks)
1848 insert_localized_app_metadata(apps)
1849 translate_per_build_anti_features(apps, apks)
1851 # Scan the archive repo for apks as well
1852 if len(repodirs) > 1:
1853 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1859 # Apply information from latest apks to the application and update dates
1860 apply_info_from_latest_apk(apps, apks + archapks)
1862 # Sort the app list by name, then the web site doesn't have to by default.
1863 # (we had to wait until we'd scanned the apks to do this, because mostly the
1864 # name comes from there!)
1865 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1867 # APKs are placed into multiple repos based on the app package, providing
1868 # per-app subscription feeds for nightly builds and things like it
1869 if config['per_app_repos']:
1870 add_apks_to_per_app_repos(repodirs[0], apks)
1871 for appid, app in apps.items():
1872 repodir = os.path.join(appid, 'fdroid', 'repo')
1874 appdict[appid] = app
1875 if os.path.isdir(repodir):
1876 index.make(appdict, [appid], apks, repodir, False)
1878 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1881 if len(repodirs) > 1:
1882 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1884 # Make the index for the main repo...
1885 index.make(apps, sortedids, apks, repodirs[0], False)
1886 make_categories_txt(repodirs[0], categories)
1888 # If there's an archive repo, make the index for it. We already scanned it
1890 if len(repodirs) > 1:
1891 index.make(apps, sortedids, archapks, repodirs[1], True)
1893 git_remote = config.get('binary_transparency_remote')
1894 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1896 btlog.make_binary_transparency_log(repodirs)
1898 if config['update_stats']:
1899 # Update known apks info...
1900 knownapks.writeifchanged()
1902 # Generate latest apps data for widget
1903 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1905 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1907 appid = line.rstrip()
1908 data += appid + "\t"
1910 data += app.Name + "\t"
1911 if app.icon is not None:
1912 data += app.icon + "\t"
1913 data += app.License + "\n"
1914 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1918 write_cache(apkcache)
1920 # Update the wiki...
1922 update_wiki(apps, sortedids, apks + archapks)
1924 logging.info(_("Finished"))
1927 if __name__ == "__main__":