3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
98 return os.path.join(repodir, "icons")
99 return os.path.join(repodir, "icons-%s" % density)
102 def get_icon_dirs(repodir):
103 for density in screen_densities:
104 yield get_icon_dir(repodir, density)
107 def get_all_icon_dirs(repodir):
108 for density in all_screen_densities:
109 yield get_icon_dir(repodir, density)
112 def update_wiki(apps, sortedids, apks):
115 :param apps: fully populated list of all applications
116 :param apks: all apks, except...
118 logging.info("Updating wiki")
120 wikiredircat = 'App Redirects'
122 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
123 path=config['wiki_path'])
124 site.login(config['wiki_user'], config['wiki_password'])
126 generated_redirects = {}
128 for appid in sortedids:
129 app = metadata.App(apps[appid])
133 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
135 for af in sorted(app.AntiFeatures):
136 wikidata += '{{AntiFeature|' + af + '}}\n'
141 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
144 app.added.strftime('%Y-%m-%d') if app.added else '',
145 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
160 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
162 wikidata += app.Summary
163 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
165 wikidata += "=Description=\n"
166 wikidata += metadata.description_wiki(app.Description) + "\n"
168 wikidata += "=Maintainer Notes=\n"
169 if app.MaintainerNotes:
170 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
171 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
173 # Get a list of all packages for this application...
175 gotcurrentver = False
179 if apk['packageName'] == appid:
180 if str(apk['versionCode']) == app.CurrentVersionCode:
183 # Include ones we can't build, as a special case...
184 for build in app.builds:
186 if build.versionCode == app.CurrentVersionCode:
188 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
189 apklist.append({'versionCode': int(build.versionCode),
190 'versionName': build.versionName,
191 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
196 if apk['versionCode'] == int(build.versionCode):
201 apklist.append({'versionCode': int(build.versionCode),
202 'versionName': build.versionName,
203 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
205 if app.CurrentVersionCode == '0':
207 # Sort with most recent first...
208 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
210 wikidata += "=Versions=\n"
211 if len(apklist) == 0:
212 wikidata += "We currently have no versions of this app available."
213 elif not gotcurrentver:
214 wikidata += "We don't have the current version of this app."
216 wikidata += "We have the current version of this app."
217 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
218 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
219 if len(app.NoSourceSince) > 0:
220 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
221 if len(app.CurrentVersion) > 0:
222 wikidata += "The current (recommended) version is " + app.CurrentVersion
223 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
226 wikidata += "==" + apk['versionName'] + "==\n"
228 if 'buildproblem' in apk:
229 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
232 wikidata += "This version is built and signed by "
234 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
236 wikidata += "the original developer.\n\n"
237 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
239 wikidata += '\n[[Category:' + wikicat + ']]\n'
240 if len(app.NoSourceSince) > 0:
241 wikidata += '\n[[Category:Apps missing source code]]\n'
242 if validapks == 0 and not app.Disabled:
243 wikidata += '\n[[Category:Apps with no packages]]\n'
244 if cantupdate and not app.Disabled:
245 wikidata += "\n[[Category:Apps we cannot update]]\n"
246 if buildfails and not app.Disabled:
247 wikidata += "\n[[Category:Apps with failing builds]]\n"
248 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
249 wikidata += '\n[[Category:Apps to Update]]\n'
251 wikidata += '\n[[Category:Apps that are disabled]]\n'
252 if app.UpdateCheckMode == 'None' and not app.Disabled:
253 wikidata += '\n[[Category:Apps with no update check]]\n'
254 for appcat in app.Categories:
255 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
257 # We can't have underscores in the page name, even if they're in
258 # the package ID, because MediaWiki messes with them...
259 pagename = appid.replace('_', ' ')
261 # Drop a trailing newline, because mediawiki is going to drop it anyway
262 # and it we don't we'll think the page has changed when it hasn't...
263 if wikidata.endswith('\n'):
264 wikidata = wikidata[:-1]
266 generated_pages[pagename] = wikidata
268 # Make a redirect from the name to the ID too, unless there's
269 # already an existing page with the name and it isn't a redirect.
271 apppagename = app.Name.replace('_', ' ')
272 apppagename = apppagename.replace('{', '')
273 apppagename = apppagename.replace('}', ' ')
274 apppagename = apppagename.replace(':', ' ')
275 apppagename = apppagename.replace('[', ' ')
276 apppagename = apppagename.replace(']', ' ')
277 # Drop double spaces caused mostly by replacing ':' above
278 apppagename = apppagename.replace(' ', ' ')
279 for expagename in site.allpages(prefix=apppagename,
280 filterredir='nonredirects',
282 if expagename == apppagename:
284 # Another reason not to make the redirect page is if the app name
285 # is the same as it's ID, because that will overwrite the real page
286 # with an redirect to itself! (Although it seems like an odd
287 # scenario this happens a lot, e.g. where there is metadata but no
288 # builds or binaries to extract a name from.
289 if apppagename == pagename:
292 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
294 for tcat, genp in [(wikicat, generated_pages),
295 (wikiredircat, generated_redirects)]:
296 catpages = site.Pages['Category:' + tcat]
298 for page in catpages:
299 existingpages.append(page.name)
300 if page.name in genp:
301 pagetxt = page.edit()
302 if pagetxt != genp[page.name]:
303 logging.debug("Updating modified page " + page.name)
304 page.save(genp[page.name], summary='Auto-updated')
306 logging.debug("Page " + page.name + " is unchanged")
308 logging.warn("Deleting page " + page.name)
309 page.delete('No longer published')
310 for pagename, text in genp.items():
311 logging.debug("Checking " + pagename)
312 if pagename not in existingpages:
313 logging.debug("Creating page " + pagename)
315 newpage = site.Pages[pagename]
316 newpage.save(text, summary='Auto-created')
317 except Exception as e:
318 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
320 # Purge server cache to ensure counts are up to date
321 site.pages['Repository Maintenance'].purge()
324 def delete_disabled_builds(apps, apkcache, repodirs):
325 """Delete disabled build outputs.
327 :param apps: list of all applications, as per metadata.read_metadata
328 :param apkcache: current apk cache information
329 :param repodirs: the repo directories to process
331 for appid, app in apps.items():
332 for build in app['builds']:
333 if not build.disable:
335 apkfilename = common.get_release_filename(app, build)
336 iconfilename = "%s.%s.png" % (
339 for repodir in repodirs:
341 os.path.join(repodir, apkfilename),
342 os.path.join(repodir, apkfilename + '.asc'),
343 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
345 for density in all_screen_densities:
346 repo_dir = get_icon_dir(repodir, density)
347 files.append(os.path.join(repo_dir, iconfilename))
350 if os.path.exists(f):
351 logging.info("Deleting disabled build output " + f)
353 if apkfilename in apkcache:
354 del apkcache[apkfilename]
357 def resize_icon(iconpath, density):
359 if not os.path.isfile(iconpath):
364 fp = open(iconpath, 'rb')
366 size = dpi_to_px(density)
368 if any(length > size for length in im.size):
370 im.thumbnail((size, size), Image.ANTIALIAS)
371 logging.debug("%s was too large at %s - new size is %s" % (
372 iconpath, oldsize, im.size))
373 im.save(iconpath, "PNG")
375 except Exception as e:
376 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
383 def resize_all_icons(repodirs):
384 """Resize all icons that exceed the max size
386 :param repodirs: the repo directories to process
388 for repodir in repodirs:
389 for density in screen_densities:
390 icon_dir = get_icon_dir(repodir, density)
391 icon_glob = os.path.join(icon_dir, '*.png')
392 for iconpath in glob.glob(icon_glob):
393 resize_icon(iconpath, density)
397 """ Get the signing certificate of an apk. To get the same md5 has that
398 Android gets, we encode the .RSA certificate in a specific format and pass
399 it hex-encoded to the md5 digest algorithm.
401 :param apkpath: path to the apk
402 :returns: A string containing the md5 of the signature of the apk or None
403 if an error occurred.
406 with zipfile.ZipFile(apkpath, 'r') as apk:
407 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
410 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
413 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
416 cert = apk.read(certs[0])
418 cert_encoded = common.get_certificate(cert)
420 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
423 def get_cache_file():
424 return os.path.join('tmp', 'apkcache')
428 """Get the cached dict of the APK index
430 Gather information about all the apk files in the repo directory,
431 using cached data if possible. Some of the index operations take a
432 long time, like calculating the SHA-256 and verifying the APK
435 The cache is invalidated if the metadata version is different, or
436 the 'allow_disabled_algorithms' config/option is different. In
437 those cases, there is no easy way to know what has changed from
438 the cache, so just rerun the whole thing.
443 apkcachefile = get_cache_file()
444 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
445 if not options.clean and os.path.exists(apkcachefile):
446 with open(apkcachefile, 'rb') as cf:
447 apkcache = pickle.load(cf, encoding='utf-8')
448 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
449 or apkcache.get('allow_disabled_algorithms') != ada:
454 apkcache["METADATA_VERSION"] = METADATA_VERSION
455 apkcache['allow_disabled_algorithms'] = ada
460 def write_cache(apkcache):
461 apkcachefile = get_cache_file()
462 cache_path = os.path.dirname(apkcachefile)
463 if not os.path.exists(cache_path):
464 os.makedirs(cache_path)
465 with open(apkcachefile, 'wb') as cf:
466 pickle.dump(apkcache, cf)
469 def get_icon_bytes(apkzip, iconsrc):
470 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
472 return apkzip.read(iconsrc)
474 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
477 def sha256sum(filename):
478 '''Calculate the sha256 of the given file'''
479 sha = hashlib.sha256()
480 with open(filename, 'rb') as f:
486 return sha.hexdigest()
489 def has_known_vulnerability(filename):
490 """checks for known vulnerabilities in the APK
492 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
493 version. Google also enforces this:
494 https://support.google.com/faqs/answer/6376725?hl=en
496 Checks whether there are more than one classes.dex or AndroidManifest.xml
497 files, which is invalid and an essential part of the "Master Key" attack.
499 http://www.saurik.com/id/17
502 # statically load this pattern
503 if not hasattr(has_known_vulnerability, "pattern"):
504 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
507 with zipfile.ZipFile(filename) as zf:
508 for name in zf.namelist():
509 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
512 chunk = lib.read(4096)
515 m = has_known_vulnerability.pattern.search(chunk)
517 version = m.group(1).decode('ascii')
518 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
519 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
520 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
521 logging.debug(_('"{path}" contains recent {name} ({version})')
522 .format(path=filename, name=name, version=version))
524 logging.warning(_('"{path}" contains outdated {name} ({version})')
525 .format(path=filename, name=name, version=version))
528 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
529 if name in files_in_apk:
531 files_in_apk.add(name)
536 def insert_obbs(repodir, apps, apks):
537 """Scans the .obb files in a given repo directory and adds them to the
538 relevant APK instances. OBB files have versionCodes like APK
539 files, and they are loosely associated. If there is an OBB file
540 present, then any APK with the same or higher versionCode will use
541 that OBB file. There are two OBB types: main and patch, each APK
542 can only have only have one of each.
544 https://developer.android.com/google/play/expansion-files.html
546 :param repodir: repo directory to scan
547 :param apps: list of current, valid apps
548 :param apks: current information on all APKs
552 def obbWarnDelete(f, msg):
553 logging.warning(msg + ' ' + f)
554 if options.delete_unknown:
555 logging.error(_("Deleting unknown file: {path}").format(path=f))
559 java_Integer_MIN_VALUE = -pow(2, 31)
560 currentPackageNames = apps.keys()
561 for f in glob.glob(os.path.join(repodir, '*.obb')):
562 obbfile = os.path.basename(f)
563 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
564 chunks = obbfile.split('.')
565 if chunks[0] != 'main' and chunks[0] != 'patch':
566 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
568 if not re.match(r'^-?[0-9]+$', chunks[1]):
569 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
570 .format(name=chunks[0]))
572 versionCode = int(chunks[1])
573 packagename = ".".join(chunks[2:-1])
575 highestVersionCode = java_Integer_MIN_VALUE
576 if packagename not in currentPackageNames:
577 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
580 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
581 highestVersionCode = apk['versionCode']
582 if versionCode > highestVersionCode:
583 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
584 .format(integer=str(versionCode)))
586 obbsha256 = sha256sum(f)
587 obbs.append((packagename, versionCode, obbfile, obbsha256))
590 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
591 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
592 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
593 apk['obbMainFile'] = obbfile
594 apk['obbMainFileSha256'] = obbsha256
595 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
596 apk['obbPatchFile'] = obbfile
597 apk['obbPatchFileSha256'] = obbsha256
598 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
602 def translate_per_build_anti_features(apps, apks):
603 """Grab the anti-features list from the build metadata
605 For most Anti-Features, they are really most applicable per-APK,
606 not for an app. An app can fix a vulnerability, add/remove
607 tracking, etc. This reads the 'antifeatures' list from the Build
608 entries in the fdroiddata metadata file, then transforms it into
609 the 'antiFeatures' list of unique items for the index.
611 The field key is all lower case in the metadata file to match the
612 rest of the Build fields. It is 'antiFeatures' camel case in the
613 implementation, index, and fdroidclient since it is translated
614 from the build 'antifeatures' field, not directly included.
618 antiFeatures = dict()
619 for packageName, app in apps.items():
621 for build in app['builds']:
622 afl = build.get('antifeatures')
624 d[int(build.versionCode)] = afl
626 antiFeatures[packageName] = d
629 d = antiFeatures.get(apk['packageName'])
631 afl = d.get(apk['versionCode'])
633 apk['antiFeatures'].update(afl)
636 def _get_localized_dict(app, locale):
637 '''get the dict to add localized store metadata to'''
638 if 'localized' not in app:
639 app['localized'] = collections.OrderedDict()
640 if locale not in app['localized']:
641 app['localized'][locale] = collections.OrderedDict()
642 return app['localized'][locale]
645 def _set_localized_text_entry(app, locale, key, f):
646 limit = config['char_limits'][key]
647 localized = _get_localized_dict(app, locale)
649 text = fp.read()[:limit]
651 localized[key] = text
654 def _set_author_entry(app, key, f):
655 limit = config['char_limits']['author']
657 text = fp.read()[:limit]
662 def copy_triple_t_store_metadata(apps):
663 """Include store metadata from the app's source repo
665 The Triple-T Gradle Play Publisher is a plugin that has a standard
666 file layout for all of the metadata and graphics that the Google
667 Play Store accepts. Since F-Droid has the git repo, it can just
668 pluck those files directly. This method reads any text files into
669 the app dict, then copies any graphics into the fdroid repo
672 This needs to be run before insert_localized_app_metadata() so that
673 the graphics files that are copied into the fdroid repo get
676 https://github.com/Triple-T/gradle-play-publisher#upload-images
677 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
681 if not os.path.isdir('build'):
682 return # nothing to do
684 for packageName, app in apps.items():
685 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
686 logging.debug('Triple-T Gradle Play Publisher: ' + d)
687 for root, dirs, files in os.walk(d):
688 segments = root.split('/')
689 locale = segments[-2]
691 if f == 'fulldescription':
692 _set_localized_text_entry(app, locale, 'description',
693 os.path.join(root, f))
695 elif f == 'shortdescription':
696 _set_localized_text_entry(app, locale, 'summary',
697 os.path.join(root, f))
700 _set_localized_text_entry(app, locale, 'name',
701 os.path.join(root, f))
704 _set_localized_text_entry(app, locale, 'video',
705 os.path.join(root, f))
707 elif f == 'whatsnew':
708 _set_localized_text_entry(app, segments[-1], 'whatsNew',
709 os.path.join(root, f))
711 elif f == 'contactEmail':
712 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
714 elif f == 'contactPhone':
715 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
717 elif f == 'contactWebsite':
718 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
721 base, extension = common.get_extension(f)
722 dirname = os.path.basename(root)
723 if extension in ALLOWED_EXTENSIONS \
724 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
725 if segments[-2] == 'listing':
726 locale = segments[-3]
728 locale = segments[-2]
729 destdir = os.path.join('repo', packageName, locale, dirname)
730 os.makedirs(destdir, mode=0o755, exist_ok=True)
731 sourcefile = os.path.join(root, f)
732 destfile = os.path.join(destdir, os.path.basename(f))
733 logging.debug('copying ' + sourcefile + ' ' + destfile)
734 shutil.copy(sourcefile, destfile)
737 def insert_localized_app_metadata(apps):
738 """scans standard locations for graphics and localized text
740 Scans for localized description files, store graphics, and
741 screenshot PNG files in statically defined screenshots directory
742 and adds them to the app metadata. The screenshots and graphic
743 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
744 and must be in the following layout:
745 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
747 repo/packageName/locale/featureGraphic.png
748 repo/packageName/locale/phoneScreenshots/1.png
749 repo/packageName/locale/phoneScreenshots/2.png
751 The changelog files must be text files named with the versionCode
752 ending with ".txt" and must be in the following layout:
753 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
755 repo/packageName/locale/changelogs/12345.txt
757 This will scan the each app's source repo then the metadata/ dir
758 for these standard locations of changelog files. If it finds
759 them, they will be added to the dict of all packages, with the
760 versions in the metadata/ folder taking precendence over the what
761 is in the app's source repo.
763 Where "packageName" is the app's packageName and "locale" is the locale
764 of the graphics, e.g. what language they are in, using the IETF RFC5646
765 format (en-US, fr-CA, es-MX, etc).
767 This will also scan the app's git for a fastlane folder, and the
768 metadata/ folder and the apps' source repos for standard locations
769 of graphic and screenshot files. If it finds them, it will copy
770 them into the repo. The fastlane files follow this pattern:
771 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
775 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
776 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
777 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
779 for srcd in sorted(sourcedirs):
780 if not os.path.isdir(srcd):
782 for root, dirs, files in os.walk(srcd):
783 segments = root.split('/')
784 packageName = segments[1]
785 if packageName not in apps:
786 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
788 locale = segments[-1]
789 destdir = os.path.join('repo', packageName, locale)
791 if f in ('description.txt', 'full_description.txt'):
792 _set_localized_text_entry(apps[packageName], locale, 'description',
793 os.path.join(root, f))
795 elif f in ('summary.txt', 'short_description.txt'):
796 _set_localized_text_entry(apps[packageName], locale, 'summary',
797 os.path.join(root, f))
799 elif f in ('name.txt', 'title.txt'):
800 _set_localized_text_entry(apps[packageName], locale, 'name',
801 os.path.join(root, f))
803 elif f == 'video.txt':
804 _set_localized_text_entry(apps[packageName], locale, 'video',
805 os.path.join(root, f))
807 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
808 locale = segments[-2]
809 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
810 os.path.join(root, f))
813 base, extension = common.get_extension(f)
814 if locale == 'images':
815 locale = segments[-2]
816 destdir = os.path.join('repo', packageName, locale)
817 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
818 os.makedirs(destdir, mode=0o755, exist_ok=True)
819 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
820 shutil.copy(os.path.join(root, f), destdir)
822 if d in SCREENSHOT_DIRS:
823 if locale == 'images':
824 locale = segments[-2]
825 destdir = os.path.join('repo', packageName, locale)
826 for f in glob.glob(os.path.join(root, d, '*.*')):
827 _ignored, extension = common.get_extension(f)
828 if extension in ALLOWED_EXTENSIONS:
829 screenshotdestdir = os.path.join(destdir, d)
830 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
831 logging.debug('copying ' + f + ' ' + screenshotdestdir)
832 shutil.copy(f, screenshotdestdir)
834 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
836 if not os.path.isdir(d):
838 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
839 if not os.path.isfile(f):
841 segments = f.split('/')
842 packageName = segments[1]
844 screenshotdir = segments[3]
845 filename = os.path.basename(f)
846 base, extension = common.get_extension(filename)
848 if packageName not in apps:
849 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
850 .format(path=filename, name=packageName))
852 graphics = _get_localized_dict(apps[packageName], locale)
854 if extension not in ALLOWED_EXTENSIONS:
855 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
856 elif base in GRAPHIC_NAMES:
857 # there can only be zero or one of these per locale
858 graphics[base] = filename
859 elif screenshotdir in SCREENSHOT_DIRS:
860 # there can any number of these per locale
861 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
862 if screenshotdir not in graphics:
863 graphics[screenshotdir] = []
864 graphics[screenshotdir].append(filename)
866 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
869 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
870 """Scan a repo for all files with an extension except APK/OBB
872 :param apkcache: current cached info about all repo files
873 :param repodir: repo directory to scan
874 :param knownapks: list of all known files, as per metadata.read_metadata
875 :param use_date_from_file: use date from file (instead of current date)
876 for newly added files
881 repodir = repodir.encode('utf-8')
882 for name in os.listdir(repodir):
883 file_extension = common.get_file_extension(name)
884 if file_extension == 'apk' or file_extension == 'obb':
886 filename = os.path.join(repodir, name)
887 name_utf8 = name.decode('utf-8')
888 if filename.endswith(b'_src.tar.gz'):
889 logging.debug(_('skipping source tarball: {path}')
890 .format(path=filename.decode('utf-8')))
892 if not common.is_repo_file(filename):
894 stat = os.stat(filename)
895 if stat.st_size == 0:
896 raise FDroidException(_('{path} is zero size!')
897 .format(path=filename))
899 shasum = sha256sum(filename)
902 repo_file = apkcache[name]
903 # added time is cached as tuple but used here as datetime instance
904 if 'added' in repo_file:
905 a = repo_file['added']
906 if isinstance(a, datetime):
907 repo_file['added'] = a
909 repo_file['added'] = datetime(*a[:6])
910 if repo_file.get('hash') == shasum:
911 logging.debug(_("Reading {apkfilename} from cache")
912 .format(apkfilename=name_utf8))
915 logging.debug(_("Ignoring stale cache data for {apkfilename}")
916 .format(apkfilename=name_utf8))
919 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
920 repo_file = collections.OrderedDict()
921 repo_file['name'] = os.path.splitext(name_utf8)[0]
922 # TODO rename apkname globally to something more generic
923 repo_file['apkName'] = name_utf8
924 repo_file['hash'] = shasum
925 repo_file['hashType'] = 'sha256'
926 repo_file['versionCode'] = 0
927 repo_file['versionName'] = shasum
928 # the static ID is the SHA256 unless it is set in the metadata
929 repo_file['packageName'] = shasum
931 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
933 repo_file['packageName'] = m.group(1)
934 repo_file['versionCode'] = int(m.group(2))
935 srcfilename = name + b'_src.tar.gz'
936 if os.path.exists(os.path.join(repodir, srcfilename)):
937 repo_file['srcname'] = srcfilename.decode('utf-8')
938 repo_file['size'] = stat.st_size
940 apkcache[name] = repo_file
943 if use_date_from_file:
944 timestamp = stat.st_ctime
945 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
947 default_date_param = None
949 # Record in knownapks, getting the added date at the same time..
950 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
951 default_date=default_date_param)
953 repo_file['added'] = added
955 repo_files.append(repo_file)
957 return repo_files, cachechanged
960 def scan_apk(apk_file):
962 Scans an APK file and returns dictionary with metadata of the APK.
964 Attention: This does *not* verify that the APK signature is correct.
966 :param apk_file: The (ideally absolute) path to the APK file
967 :raises BuildException
968 :return A dict containing APK metadata
971 'hash': sha256sum(apk_file),
972 'hashType': 'sha256',
973 'uses-permission': [],
974 'uses-permission-sdk-23': [],
978 'antiFeatures': set(),
981 if SdkToolsPopen(['aapt', 'version'], output=False):
982 scan_apk_aapt(apk, apk_file)
984 scan_apk_androguard(apk, apk_file)
986 # Get the signature, or rather the signing key fingerprints
987 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
988 apk['sig'] = getsig(apk_file)
990 raise BuildException("Failed to get apk signature")
991 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
993 if not apk.get('signer'):
994 raise BuildException("Failed to get apk signing key fingerprint")
996 # Get size of the APK
997 apk['size'] = os.path.getsize(apk_file)
999 if 'minSdkVersion' not in apk:
1000 logging.warning("No SDK version information found in {0}".format(apk_file))
1001 apk['minSdkVersion'] = 1
1003 # Check for known vulnerabilities
1004 if has_known_vulnerability(apk_file):
1005 apk['antiFeatures'].add('KnownVuln')
1010 def scan_apk_aapt(apk, apkfile):
1011 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1012 if p.returncode != 0:
1013 if options.delete_unknown:
1014 if os.path.exists(apkfile):
1015 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1018 logging.error("Could not find {0} to remove it".format(apkfile))
1020 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1021 raise BuildException(_("Invalid APK"))
1022 for line in p.output.splitlines():
1023 if line.startswith("package:"):
1025 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1026 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1027 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1028 except Exception as e:
1029 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1030 elif line.startswith("application:"):
1031 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1032 # Keep path to non-dpi icon in case we need it
1033 match = re.match(APK_ICON_PAT_NODPI, line)
1035 apk['icons_src']['-1'] = match.group(1)
1036 elif line.startswith("launchable-activity:"):
1037 # Only use launchable-activity as fallback to application
1039 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1040 if '-1' not in apk['icons_src']:
1041 match = re.match(APK_ICON_PAT_NODPI, line)
1043 apk['icons_src']['-1'] = match.group(1)
1044 elif line.startswith("application-icon-"):
1045 match = re.match(APK_ICON_PAT, line)
1047 density = match.group(1)
1048 path = match.group(2)
1049 apk['icons_src'][density] = path
1050 elif line.startswith("sdkVersion:"):
1051 m = re.match(APK_SDK_VERSION_PAT, line)
1053 logging.error(line.replace('sdkVersion:', '')
1054 + ' is not a valid minSdkVersion!')
1056 apk['minSdkVersion'] = m.group(1)
1057 # if target not set, default to min
1058 if 'targetSdkVersion' not in apk:
1059 apk['targetSdkVersion'] = m.group(1)
1060 elif line.startswith("targetSdkVersion:"):
1061 m = re.match(APK_SDK_VERSION_PAT, line)
1063 logging.error(line.replace('targetSdkVersion:', '')
1064 + ' is not a valid targetSdkVersion!')
1066 apk['targetSdkVersion'] = m.group(1)
1067 elif line.startswith("maxSdkVersion:"):
1068 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1069 elif line.startswith("native-code:"):
1070 apk['nativecode'] = []
1071 for arch in line[13:].split(' '):
1072 apk['nativecode'].append(arch[1:-1])
1073 elif line.startswith('uses-permission:'):
1074 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1075 if perm_match['maxSdkVersion']:
1076 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1077 permission = UsesPermission(
1079 perm_match['maxSdkVersion']
1082 apk['uses-permission'].append(permission)
1083 elif line.startswith('uses-permission-sdk-23:'):
1084 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1085 if perm_match['maxSdkVersion']:
1086 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1087 permission_sdk_23 = UsesPermissionSdk23(
1089 perm_match['maxSdkVersion']
1092 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1094 elif line.startswith('uses-feature:'):
1095 feature = re.match(APK_FEATURE_PAT, line).group(1)
1096 # Filter out this, it's only added with the latest SDK tools and
1097 # causes problems for lots of apps.
1098 if feature != "android.hardware.screen.portrait" \
1099 and feature != "android.hardware.screen.landscape":
1100 if feature.startswith("android.feature."):
1101 feature = feature[16:]
1102 apk['features'].add(feature)
1105 def scan_apk_androguard(apk, apkfile):
1107 from androguard.core.bytecodes.apk import APK
1108 apkobject = APK(apkfile)
1109 if apkobject.is_valid_APK():
1110 arsc = apkobject.get_android_resources()
1112 if options.delete_unknown:
1113 if os.path.exists(apkfile):
1114 logging.error(_("Failed to get apk information, deleting {path}")
1115 .format(path=apkfile))
1118 logging.error(_("Could not find {path} to remove it")
1119 .format(path=apkfile))
1121 logging.error(_("Failed to get apk information, skipping {path}")
1122 .format(path=apkfile))
1123 raise BuildException(_("Invalid APK"))
1125 raise FDroidException("androguard library is not installed and aapt not present")
1126 except FileNotFoundError:
1127 logging.error(_("Could not open apk file for analysis"))
1128 raise BuildException(_("Invalid APK"))
1130 apk['packageName'] = apkobject.get_package()
1131 apk['versionCode'] = int(apkobject.get_androidversion_code())
1132 apk['versionName'] = apkobject.get_androidversion_name()
1133 if apk['versionName'][0] == "@":
1134 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1135 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1136 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1137 apk['name'] = apkobject.get_app_name()
1139 if apkobject.get_max_sdk_version() is not None:
1140 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1141 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1142 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1144 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1145 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1147 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1149 for file in apkobject.get_files():
1150 d_re = density_re.match(file)
1152 folder = d_re.group(1).split('-')
1154 resolution = folder[1]
1157 density = screen_resolutions[resolution]
1158 apk['icons_src'][density] = d_re.group(0)
1160 if apk['icons_src'].get('-1') is None:
1161 apk['icons_src']['-1'] = apk['icons_src']['160']
1163 arch_re = re.compile("^lib/(.*)/.*$")
1164 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1166 apk['nativecode'] = []
1167 apk['nativecode'].extend(sorted(list(arch)))
1169 xml = apkobject.get_android_manifest_xml()
1171 for item in xml.getElementsByTagName('uses-permission'):
1172 name = str(item.getAttribute("android:name"))
1173 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1174 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1175 permission = UsesPermission(
1179 apk['uses-permission'].append(permission)
1181 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1182 name = str(item.getAttribute("android:name"))
1183 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1184 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1185 permission_sdk_23 = UsesPermissionSdk23(
1189 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1191 for item in xml.getElementsByTagName('uses-feature'):
1192 feature = str(item.getAttribute("android:name"))
1193 if feature != "android.hardware.screen.portrait" \
1194 and feature != "android.hardware.screen.landscape":
1195 if feature.startswith("android.feature."):
1196 feature = feature[16:]
1197 apk['features'].append(feature)
1200 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1201 allow_disabled_algorithms=False, archive_bad_sig=False):
1202 """Processes the apk with the given filename in the given repo directory.
1204 This also extracts the icons.
1206 :param apkcache: current apk cache information
1207 :param apkfilename: the filename of the apk to scan
1208 :param repodir: repo directory to scan
1209 :param knownapks: known apks info
1210 :param use_date_from_apk: use date from APK (instead of current date)
1211 for newly added APKs
1212 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1213 disabled algorithms in the signature (e.g. MD5)
1214 :param archive_bad_sig: move APKs with a bad signature to the archive
1215 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1216 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1220 apkfile = os.path.join(repodir, apkfilename)
1222 cachechanged = False
1224 if apkfilename in apkcache:
1225 apk = apkcache[apkfilename]
1226 if apk.get('hash') == sha256sum(apkfile):
1227 logging.debug(_("Reading {apkfilename} from cache")
1228 .format(apkfilename=apkfilename))
1231 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1232 .format(apkfilename=apkfilename))
1235 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1238 apk = scan_apk(apkfile)
1239 except BuildException:
1240 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1241 .format(apkfilename=apkfilename))
1242 return True, None, False
1244 # Check for debuggable apks...
1245 if common.isApkAndDebuggable(apkfile):
1246 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1248 if options.rename_apks:
1249 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1250 std_short_name = os.path.join(repodir, n)
1251 if apkfile != std_short_name:
1252 if os.path.exists(std_short_name):
1253 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1254 if apkfile != std_long_name:
1255 if os.path.exists(std_long_name):
1256 dupdir = os.path.join('duplicates', repodir)
1257 if not os.path.isdir(dupdir):
1258 os.makedirs(dupdir, exist_ok=True)
1259 dupfile = os.path.join('duplicates', std_long_name)
1260 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1261 os.rename(apkfile, dupfile)
1262 return True, None, False
1264 os.rename(apkfile, std_long_name)
1265 apkfile = std_long_name
1267 os.rename(apkfile, std_short_name)
1268 apkfile = std_short_name
1269 apkfilename = apkfile[len(repodir) + 1:]
1271 apk['apkName'] = apkfilename
1272 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1273 if os.path.exists(os.path.join(repodir, srcfilename)):
1274 apk['srcname'] = srcfilename
1276 # verify the jar signature is correct, allow deprecated
1277 # algorithms only if the APK is in the archive.
1279 if not common.verify_apk_signature(apkfile):
1280 if repodir == 'archive' or allow_disabled_algorithms:
1281 if common.verify_old_apk_signature(apkfile):
1282 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1290 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1291 .format(apkfilename=apkfilename))
1292 move_apk_between_sections(repodir, 'archive', apk)
1294 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1295 .format(apkfilename=apkfilename))
1296 return True, None, False
1298 apkzip = zipfile.ZipFile(apkfile, 'r')
1300 manifest = apkzip.getinfo('AndroidManifest.xml')
1301 if manifest.date_time[1] == 0: # month can't be zero
1302 logging.debug(_('AndroidManifest.xml has no date'))
1304 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1306 # extract icons from APK zip file
1307 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1309 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1311 apkzip.close() # ensure that APK zip file gets closed
1313 # resize existing icons for densities missing in the APK
1314 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1316 if use_date_from_apk and manifest.date_time[1] != 0:
1317 default_date_param = datetime(*manifest.date_time)
1319 default_date_param = None
1321 # Record in known apks, getting the added date at the same time..
1322 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1323 default_date=default_date_param)
1325 apk['added'] = added
1327 apkcache[apkfilename] = apk
1330 return False, apk, cachechanged
1333 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1334 """Processes the apks in the given repo directory.
1336 This also extracts the icons.
1338 :param apkcache: current apk cache information
1339 :param repodir: repo directory to scan
1340 :param knownapks: known apks info
1341 :param use_date_from_apk: use date from APK (instead of current date)
1342 for newly added APKs
1343 :returns: (apks, cachechanged) where apks is a list of apk information,
1344 and cachechanged is True if the apkcache got changed.
1347 cachechanged = False
1349 for icon_dir in get_all_icon_dirs(repodir):
1350 if os.path.exists(icon_dir):
1352 shutil.rmtree(icon_dir)
1353 os.makedirs(icon_dir)
1355 os.makedirs(icon_dir)
1358 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1359 apkfilename = apkfile[len(repodir) + 1:]
1360 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1361 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1362 use_date_from_apk, ada, True)
1366 cachechanged = cachechanged or cachethis
1368 return apks, cachechanged
1371 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1373 Extracts icons from the given APK zip in various densities,
1374 saves them into given repo directory
1375 and stores their names in the APK metadata dictionary.
1377 :param icon_filename: A string representing the icon's file name
1378 :param apk: A populated dictionary containing APK metadata.
1379 Needs to have 'icons_src' key
1380 :param apkzip: An opened zipfile.ZipFile of the APK file
1381 :param repo_dir: The directory of the APK's repository
1382 :return: A list of icon densities that are missing
1384 empty_densities = []
1385 for density in screen_densities:
1386 if density not in apk['icons_src']:
1387 empty_densities.append(density)
1389 icon_src = apk['icons_src'][density]
1390 icon_dir = get_icon_dir(repo_dir, density)
1391 icon_dest = os.path.join(icon_dir, icon_filename)
1393 # Extract the icon files per density
1394 if icon_src.endswith('.xml'):
1395 png = os.path.basename(icon_src)[:-4] + '.png'
1396 for f in apkzip.namelist():
1398 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1399 if m and screen_resolutions[m.group(2)] == density:
1401 if icon_src.endswith('.xml'):
1402 empty_densities.append(density)
1405 with open(icon_dest, 'wb') as f:
1406 f.write(get_icon_bytes(apkzip, icon_src))
1407 apk['icons'][density] = icon_filename
1408 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1409 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1410 del apk['icons_src'][density]
1411 empty_densities.append(density)
1413 if '-1' in apk['icons_src']:
1414 icon_src = apk['icons_src']['-1']
1415 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1416 with open(icon_path, 'wb') as f:
1417 f.write(get_icon_bytes(apkzip, icon_src))
1419 im = Image.open(icon_path)
1420 dpi = px_to_dpi(im.size[0])
1421 for density in screen_densities:
1422 if density in apk['icons']:
1424 if density == screen_densities[-1] or dpi >= int(density):
1425 apk['icons'][density] = icon_filename
1426 shutil.move(icon_path,
1427 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1428 empty_densities.remove(density)
1430 except Exception as e:
1431 logging.warning(_("Failed reading {path}: {error}")
1432 .format(path=icon_path, error=e))
1435 apk['icon'] = icon_filename
1437 return empty_densities
1440 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1442 Resize existing icons for densities missing in the APK to ensure all densities are available
1444 :param empty_densities: A list of icon densities that are missing
1445 :param icon_filename: A string representing the icon's file name
1446 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1447 :param repo_dir: The directory of the APK's repository
1449 # First try resizing down to not lose quality
1451 for density in screen_densities:
1452 if density not in empty_densities:
1453 last_density = density
1455 if last_density is None:
1457 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1459 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1460 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1463 fp = open(last_icon_path, 'rb')
1466 size = dpi_to_px(density)
1468 im.thumbnail((size, size), Image.ANTIALIAS)
1469 im.save(icon_path, "PNG")
1470 empty_densities.remove(density)
1471 except Exception as e:
1472 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1477 # Then just copy from the highest resolution available
1479 for density in reversed(screen_densities):
1480 if density not in empty_densities:
1481 last_density = density
1484 if last_density is None:
1488 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1489 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1491 empty_densities.remove(density)
1493 for density in screen_densities:
1494 icon_dir = get_icon_dir(repo_dir, density)
1495 icon_dest = os.path.join(icon_dir, icon_filename)
1496 resize_icon(icon_dest, density)
1498 # Copy from icons-mdpi to icons since mdpi is the baseline density
1499 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1500 if os.path.isfile(baseline):
1501 apk['icons']['0'] = icon_filename
1502 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1505 def apply_info_from_latest_apk(apps, apks):
1507 Some information from the apks needs to be applied up to the application level.
1508 When doing this, we use the info from the most recent version's apk.
1509 We deal with figuring out when the app was added and last updated at the same time.
1511 for appid, app in apps.items():
1512 bestver = UNSET_VERSION_CODE
1514 if apk['packageName'] == appid:
1515 if apk['versionCode'] > bestver:
1516 bestver = apk['versionCode']
1520 if not app.added or apk['added'] < app.added:
1521 app.added = apk['added']
1522 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1523 app.lastUpdated = apk['added']
1526 logging.debug("Don't know when " + appid + " was added")
1527 if not app.lastUpdated:
1528 logging.debug("Don't know when " + appid + " was last updated")
1530 if bestver == UNSET_VERSION_CODE:
1532 if app.Name is None:
1533 app.Name = app.AutoName or appid
1535 logging.debug("Application " + appid + " has no packages")
1537 if app.Name is None:
1538 app.Name = bestapk['name']
1539 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1540 if app.CurrentVersionCode is None:
1541 app.CurrentVersionCode = str(bestver)
1544 def make_categories_txt(repodir, categories):
1545 '''Write a category list in the repo to allow quick access'''
1547 for cat in sorted(categories):
1548 catdata += cat + '\n'
1549 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1553 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1555 def filter_apk_list_sorted(apk_list):
1557 for apk in apk_list:
1558 if apk['packageName'] == appid:
1561 # Sort the apk list by version code. First is highest/newest.
1562 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1564 for appid, app in apps.items():
1566 if app.ArchivePolicy:
1567 keepversions = int(app.ArchivePolicy[:-9])
1569 keepversions = defaultkeepversions
1571 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1572 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1574 current_app_apks = filter_apk_list_sorted(apks)
1575 if len(current_app_apks) > keepversions:
1576 # Move back the ones we don't want.
1577 for apk in current_app_apks[keepversions:]:
1578 move_apk_between_sections(repodir, archivedir, apk)
1579 archapks.append(apk)
1582 current_app_archapks = filter_apk_list_sorted(archapks)
1583 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1585 # Move forward the ones we want again, except DisableAlgorithm
1586 for apk in current_app_archapks:
1587 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1588 move_apk_between_sections(archivedir, repodir, apk)
1589 archapks.remove(apk)
1592 if kept == keepversions:
1596 def move_apk_between_sections(from_dir, to_dir, apk):
1597 """move an APK from repo to archive or vice versa"""
1599 def _move_file(from_dir, to_dir, filename, ignore_missing):
1600 from_path = os.path.join(from_dir, filename)
1601 if ignore_missing and not os.path.exists(from_path):
1603 to_path = os.path.join(to_dir, filename)
1604 if not os.path.exists(to_dir):
1606 shutil.move(from_path, to_path)
1608 if from_dir == to_dir:
1611 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1612 _move_file(from_dir, to_dir, apk['apkName'], False)
1613 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1614 for density in all_screen_densities:
1615 from_icon_dir = get_icon_dir(from_dir, density)
1616 to_icon_dir = get_icon_dir(to_dir, density)
1617 if density not in apk.get('icons', []):
1619 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1620 if 'srcname' in apk:
1621 _move_file(from_dir, to_dir, apk['srcname'], False)
1624 def add_apks_to_per_app_repos(repodir, apks):
1625 apks_per_app = dict()
1627 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1628 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1629 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1630 apks_per_app[apk['packageName']] = apk
1632 if not os.path.exists(apk['per_app_icons']):
1633 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1634 os.makedirs(apk['per_app_icons'])
1636 apkpath = os.path.join(repodir, apk['apkName'])
1637 shutil.copy(apkpath, apk['per_app_repo'])
1638 apksigpath = apkpath + '.sig'
1639 if os.path.exists(apksigpath):
1640 shutil.copy(apksigpath, apk['per_app_repo'])
1641 apkascpath = apkpath + '.asc'
1642 if os.path.exists(apkascpath):
1643 shutil.copy(apkascpath, apk['per_app_repo'])
1646 def create_metadata_from_template(apk):
1647 '''create a new metadata file using internal or external template
1649 Generate warnings for apk's with no metadata (or create skeleton
1650 metadata files, if requested on the command line). Though the
1651 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1652 since those impose things on the metadata file made from the
1653 template: field sort order, empty field value, formatting, etc.
1657 if os.path.exists('template.yml'):
1658 with open('template.yml') as f:
1660 if 'name' in apk and apk['name'] != '':
1661 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1662 r'\1 ' + apk['name'],
1664 flags=re.IGNORECASE | re.MULTILINE)
1666 logging.warning(_('{appid} does not have a name! Using package name instead.')
1667 .format(appid=apk['packageName']))
1668 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1669 r'\1 ' + apk['packageName'],
1671 flags=re.IGNORECASE | re.MULTILINE)
1672 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1676 app['Categories'] = [os.path.basename(os.getcwd())]
1677 # include some blanks as part of the template
1678 app['AuthorName'] = ''
1681 app['IssueTracker'] = ''
1682 app['SourceCode'] = ''
1683 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1684 if 'name' in apk and apk['name'] != '':
1685 app['Name'] = apk['name']
1687 logging.warning(_('{appid} does not have a name! Using package name instead.')
1688 .format(appid=apk['packageName']))
1689 app['Name'] = apk['packageName']
1690 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1691 yaml.dump(app, f, default_flow_style=False)
1692 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1701 global config, options
1703 # Parse command line...
1704 parser = ArgumentParser()
1705 common.setup_global_opts(parser)
1706 parser.add_argument("--create-key", action="store_true", default=False,
1707 help=_("Add a repo signing key to an unsigned repo"))
1708 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1709 help=_("Add skeleton metadata files for APKs that are missing them"))
1710 parser.add_argument("--delete-unknown", action="store_true", default=False,
1711 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1712 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1713 help=_("Report on build data status"))
1714 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1715 help=_("Interactively ask about things that need updating."))
1716 parser.add_argument("-I", "--icons", action="store_true", default=False,
1717 help=_("Resize all the icons exceeding the max pixel size and exit"))
1718 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1719 help=_("Specify editor to use in interactive mode. Default " +
1720 "is {path}").format(path='/etc/alternatives/editor'))
1721 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1722 help=_("Update the wiki"))
1723 parser.add_argument("--pretty", action="store_true", default=False,
1724 help=_("Produce human-readable XML/JSON for index files"))
1725 parser.add_argument("--clean", action="store_true", default=False,
1726 help=_("Clean update - don't uses caches, reprocess all APKs"))
1727 parser.add_argument("--nosign", action="store_true", default=False,
1728 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1729 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1730 help=_("Use date from APK instead of current time for newly added APKs"))
1731 parser.add_argument("--rename-apks", action="store_true", default=False,
1732 help=_("Rename APK files that do not match package.name_123.apk"))
1733 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1734 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1735 metadata.add_metadata_arguments(parser)
1736 options = parser.parse_args()
1737 metadata.warnings_action = options.W
1739 config = common.read_config(options)
1741 if not ('jarsigner' in config and 'keytool' in config):
1742 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1745 if config['archive_older'] != 0:
1746 repodirs.append('archive')
1747 if not os.path.exists('archive'):
1751 resize_all_icons(repodirs)
1754 if options.rename_apks:
1755 options.clean = True
1757 # check that icons exist now, rather than fail at the end of `fdroid update`
1758 for k in ['repo_icon', 'archive_icon']:
1760 if not os.path.exists(config[k]):
1761 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1762 .format(name=k, path=config[k]))
1765 # if the user asks to create a keystore, do it now, reusing whatever it can
1766 if options.create_key:
1767 if os.path.exists(config['keystore']):
1768 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1769 logging.critical("\t'" + config['keystore'] + "'")
1772 if 'repo_keyalias' not in config:
1773 config['repo_keyalias'] = socket.getfqdn()
1774 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1775 if 'keydname' not in config:
1776 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1777 common.write_to_config(config, 'keydname', config['keydname'])
1778 if 'keystore' not in config:
1779 config['keystore'] = common.default_config['keystore']
1780 common.write_to_config(config, 'keystore', config['keystore'])
1782 password = common.genpassword()
1783 if 'keystorepass' not in config:
1784 config['keystorepass'] = password
1785 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1786 if 'keypass' not in config:
1787 config['keypass'] = password
1788 common.write_to_config(config, 'keypass', config['keypass'])
1789 common.genkeystore(config)
1792 apps = metadata.read_metadata()
1794 # Generate a list of categories...
1796 for app in apps.values():
1797 categories.update(app.Categories)
1799 # Read known apks data (will be updated and written back when we've finished)
1800 knownapks = common.KnownApks()
1803 apkcache = get_cache()
1805 # Delete builds for disabled apps
1806 delete_disabled_builds(apps, apkcache, repodirs)
1808 # Scan all apks in the main repo
1809 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1811 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1812 options.use_date_from_apk)
1813 cachechanged = cachechanged or fcachechanged
1816 if apk['packageName'] not in apps:
1817 if options.create_metadata:
1818 create_metadata_from_template(apk)
1819 apps = metadata.read_metadata()
1821 msg = _("{apkfilename} ({appid}) has no metadata!") \
1822 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1823 if options.delete_unknown:
1824 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1825 .format(apkfilename=apk['apkName']))
1826 rmf = os.path.join(repodirs[0], apk['apkName'])
1827 if not os.path.exists(rmf):
1828 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1832 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1834 copy_triple_t_store_metadata(apps)
1835 insert_obbs(repodirs[0], apps, apks)
1836 insert_localized_app_metadata(apps)
1837 translate_per_build_anti_features(apps, apks)
1839 # Scan the archive repo for apks as well
1840 if len(repodirs) > 1:
1841 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1847 # Apply information from latest apks to the application and update dates
1848 apply_info_from_latest_apk(apps, apks + archapks)
1850 # Sort the app list by name, then the web site doesn't have to by default.
1851 # (we had to wait until we'd scanned the apks to do this, because mostly the
1852 # name comes from there!)
1853 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1855 # APKs are placed into multiple repos based on the app package, providing
1856 # per-app subscription feeds for nightly builds and things like it
1857 if config['per_app_repos']:
1858 add_apks_to_per_app_repos(repodirs[0], apks)
1859 for appid, app in apps.items():
1860 repodir = os.path.join(appid, 'fdroid', 'repo')
1862 appdict[appid] = app
1863 if os.path.isdir(repodir):
1864 index.make(appdict, [appid], apks, repodir, False)
1866 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1869 if len(repodirs) > 1:
1870 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1872 # Make the index for the main repo...
1873 index.make(apps, sortedids, apks, repodirs[0], False)
1874 make_categories_txt(repodirs[0], categories)
1876 # If there's an archive repo, make the index for it. We already scanned it
1878 if len(repodirs) > 1:
1879 index.make(apps, sortedids, archapks, repodirs[1], True)
1881 git_remote = config.get('binary_transparency_remote')
1882 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1884 btlog.make_binary_transparency_log(repodirs)
1886 if config['update_stats']:
1887 # Update known apks info...
1888 knownapks.writeifchanged()
1890 # Generate latest apps data for widget
1891 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1893 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1895 appid = line.rstrip()
1896 data += appid + "\t"
1898 data += app.Name + "\t"
1899 if app.icon is not None:
1900 data += app.icon + "\t"
1901 data += app.License + "\n"
1902 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1906 write_cache(apkcache)
1908 # Update the wiki...
1910 update_wiki(apps, sortedids, apks + archapks)
1912 logging.info(_("Finished"))
1915 if __name__ == "__main__":