3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime, timedelta
33 from argparse import ArgumentParser
36 from binascii import hexlify
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
98 return os.path.join(repodir, "icons")
99 return os.path.join(repodir, "icons-%s" % density)
102 def get_icon_dirs(repodir):
103 for density in screen_densities:
104 yield get_icon_dir(repodir, density)
107 def get_all_icon_dirs(repodir):
108 for density in all_screen_densities:
109 yield get_icon_dir(repodir, density)
112 def update_wiki(apps, sortedids, apks):
115 :param apps: fully populated list of all applications
116 :param apks: all apks, except...
118 logging.info("Updating wiki")
120 wikiredircat = 'App Redirects'
122 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
123 path=config['wiki_path'])
124 site.login(config['wiki_user'], config['wiki_password'])
126 generated_redirects = {}
128 for appid in sortedids:
129 app = metadata.App(apps[appid])
133 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
135 for af in sorted(app.AntiFeatures):
136 wikidata += '{{AntiFeature|' + af + '}}\n'
141 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
144 app.added.strftime('%Y-%m-%d') if app.added else '',
145 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
160 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
162 wikidata += app.Summary
163 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
165 wikidata += "=Description=\n"
166 wikidata += metadata.description_wiki(app.Description) + "\n"
168 wikidata += "=Maintainer Notes=\n"
169 if app.MaintainerNotes:
170 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
171 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
173 # Get a list of all packages for this application...
175 gotcurrentver = False
179 if apk['packageName'] == appid:
180 if str(apk['versionCode']) == app.CurrentVersionCode:
183 # Include ones we can't build, as a special case...
184 for build in app.builds:
186 if build.versionCode == app.CurrentVersionCode:
188 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
189 apklist.append({'versionCode': int(build.versionCode),
190 'versionName': build.versionName,
191 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
196 if apk['versionCode'] == int(build.versionCode):
201 apklist.append({'versionCode': int(build.versionCode),
202 'versionName': build.versionName,
203 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
205 if app.CurrentVersionCode == '0':
207 # Sort with most recent first...
208 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
210 wikidata += "=Versions=\n"
211 if len(apklist) == 0:
212 wikidata += "We currently have no versions of this app available."
213 elif not gotcurrentver:
214 wikidata += "We don't have the current version of this app."
216 wikidata += "We have the current version of this app."
217 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
218 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
219 if len(app.NoSourceSince) > 0:
220 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
221 if len(app.CurrentVersion) > 0:
222 wikidata += "The current (recommended) version is " + app.CurrentVersion
223 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
226 wikidata += "==" + apk['versionName'] + "==\n"
228 if 'buildproblem' in apk:
229 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
232 wikidata += "This version is built and signed by "
234 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
236 wikidata += "the original developer.\n\n"
237 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
239 wikidata += '\n[[Category:' + wikicat + ']]\n'
240 if len(app.NoSourceSince) > 0:
241 wikidata += '\n[[Category:Apps missing source code]]\n'
242 if validapks == 0 and not app.Disabled:
243 wikidata += '\n[[Category:Apps with no packages]]\n'
244 if cantupdate and not app.Disabled:
245 wikidata += "\n[[Category:Apps we cannot update]]\n"
246 if buildfails and not app.Disabled:
247 wikidata += "\n[[Category:Apps with failing builds]]\n"
248 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
249 wikidata += '\n[[Category:Apps to Update]]\n'
251 wikidata += '\n[[Category:Apps that are disabled]]\n'
252 if app.UpdateCheckMode == 'None' and not app.Disabled:
253 wikidata += '\n[[Category:Apps with no update check]]\n'
254 for appcat in app.Categories:
255 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
257 # We can't have underscores in the page name, even if they're in
258 # the package ID, because MediaWiki messes with them...
259 pagename = appid.replace('_', ' ')
261 # Drop a trailing newline, because mediawiki is going to drop it anyway
262 # and it we don't we'll think the page has changed when it hasn't...
263 if wikidata.endswith('\n'):
264 wikidata = wikidata[:-1]
266 generated_pages[pagename] = wikidata
268 # Make a redirect from the name to the ID too, unless there's
269 # already an existing page with the name and it isn't a redirect.
271 apppagename = app.Name.replace('_', ' ')
272 apppagename = apppagename.replace('{', '')
273 apppagename = apppagename.replace('}', ' ')
274 apppagename = apppagename.replace(':', ' ')
275 apppagename = apppagename.replace('[', ' ')
276 apppagename = apppagename.replace(']', ' ')
277 # Drop double spaces caused mostly by replacing ':' above
278 apppagename = apppagename.replace(' ', ' ')
279 for expagename in site.allpages(prefix=apppagename,
280 filterredir='nonredirects',
282 if expagename == apppagename:
284 # Another reason not to make the redirect page is if the app name
285 # is the same as it's ID, because that will overwrite the real page
286 # with an redirect to itself! (Although it seems like an odd
287 # scenario this happens a lot, e.g. where there is metadata but no
288 # builds or binaries to extract a name from.
289 if apppagename == pagename:
292 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
294 for tcat, genp in [(wikicat, generated_pages),
295 (wikiredircat, generated_redirects)]:
296 catpages = site.Pages['Category:' + tcat]
298 for page in catpages:
299 existingpages.append(page.name)
300 if page.name in genp:
301 pagetxt = page.edit()
302 if pagetxt != genp[page.name]:
303 logging.debug("Updating modified page " + page.name)
304 page.save(genp[page.name], summary='Auto-updated')
306 logging.debug("Page " + page.name + " is unchanged")
308 logging.warn("Deleting page " + page.name)
309 page.delete('No longer published')
310 for pagename, text in genp.items():
311 logging.debug("Checking " + pagename)
312 if pagename not in existingpages:
313 logging.debug("Creating page " + pagename)
315 newpage = site.Pages[pagename]
316 newpage.save(text, summary='Auto-created')
317 except Exception as e:
318 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
320 # Purge server cache to ensure counts are up to date
321 site.pages['Repository Maintenance'].purge()
324 def delete_disabled_builds(apps, apkcache, repodirs):
325 """Delete disabled build outputs.
327 :param apps: list of all applications, as per metadata.read_metadata
328 :param apkcache: current apk cache information
329 :param repodirs: the repo directories to process
331 for appid, app in apps.items():
332 for build in app['builds']:
333 if not build.disable:
335 apkfilename = common.get_release_filename(app, build)
336 iconfilename = "%s.%s.png" % (
339 for repodir in repodirs:
341 os.path.join(repodir, apkfilename),
342 os.path.join(repodir, apkfilename + '.asc'),
343 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
345 for density in all_screen_densities:
346 repo_dir = get_icon_dir(repodir, density)
347 files.append(os.path.join(repo_dir, iconfilename))
350 if os.path.exists(f):
351 logging.info("Deleting disabled build output " + f)
353 if apkfilename in apkcache:
354 del apkcache[apkfilename]
357 def resize_icon(iconpath, density):
359 if not os.path.isfile(iconpath):
364 fp = open(iconpath, 'rb')
366 size = dpi_to_px(density)
368 if any(length > size for length in im.size):
370 im.thumbnail((size, size), Image.ANTIALIAS)
371 logging.debug("%s was too large at %s - new size is %s" % (
372 iconpath, oldsize, im.size))
373 im.save(iconpath, "PNG")
375 except Exception as e:
376 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
383 def resize_all_icons(repodirs):
384 """Resize all icons that exceed the max size
386 :param repodirs: the repo directories to process
388 for repodir in repodirs:
389 for density in screen_densities:
390 icon_dir = get_icon_dir(repodir, density)
391 icon_glob = os.path.join(icon_dir, '*.png')
392 for iconpath in glob.glob(icon_glob):
393 resize_icon(iconpath, density)
397 """ Get the signing certificate of an apk. To get the same md5 has that
398 Android gets, we encode the .RSA certificate in a specific format and pass
399 it hex-encoded to the md5 digest algorithm.
401 :param apkpath: path to the apk
402 :returns: A string containing the md5 of the signature of the apk or None
403 if an error occurred.
406 with zipfile.ZipFile(apkpath, 'r') as apk:
407 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
410 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
413 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
416 cert = apk.read(certs[0])
418 cert_encoded = common.get_certificate(cert)
420 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
423 def get_cache_file():
424 return os.path.join('tmp', 'apkcache')
428 """Get the cached dict of the APK index
430 Gather information about all the apk files in the repo directory,
431 using cached data if possible. Some of the index operations take a
432 long time, like calculating the SHA-256 and verifying the APK
435 The cache is invalidated if the metadata version is different, or
436 the 'allow_disabled_algorithms' config/option is different. In
437 those cases, there is no easy way to know what has changed from
438 the cache, so just rerun the whole thing.
443 apkcachefile = get_cache_file()
444 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
445 if not options.clean and os.path.exists(apkcachefile):
446 with open(apkcachefile, 'rb') as cf:
447 apkcache = pickle.load(cf, encoding='utf-8')
448 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
449 or apkcache.get('allow_disabled_algorithms') != ada:
454 apkcache["METADATA_VERSION"] = METADATA_VERSION
455 apkcache['allow_disabled_algorithms'] = ada
460 def write_cache(apkcache):
461 apkcachefile = get_cache_file()
462 cache_path = os.path.dirname(apkcachefile)
463 if not os.path.exists(cache_path):
464 os.makedirs(cache_path)
465 with open(apkcachefile, 'wb') as cf:
466 pickle.dump(apkcache, cf)
469 def get_icon_bytes(apkzip, iconsrc):
470 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
472 return apkzip.read(iconsrc)
474 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
477 def sha256sum(filename):
478 '''Calculate the sha256 of the given file'''
479 sha = hashlib.sha256()
480 with open(filename, 'rb') as f:
486 return sha.hexdigest()
489 def has_known_vulnerability(filename):
490 """checks for known vulnerabilities in the APK
492 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
493 version. Google also enforces this:
494 https://support.google.com/faqs/answer/6376725?hl=en
496 Checks whether there are more than one classes.dex or AndroidManifest.xml
497 files, which is invalid and an essential part of the "Master Key" attack.
499 http://www.saurik.com/id/17
502 # statically load this pattern
503 if not hasattr(has_known_vulnerability, "pattern"):
504 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
507 with zipfile.ZipFile(filename) as zf:
508 for name in zf.namelist():
509 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
512 chunk = lib.read(4096)
515 m = has_known_vulnerability.pattern.search(chunk)
517 version = m.group(1).decode('ascii')
518 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
519 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
520 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
521 logging.debug(_('"{path}" contains recent {name} ({version})')
522 .format(path=filename, name=name, version=version))
524 logging.warning(_('"{path}" contains outdated {name} ({version})')
525 .format(path=filename, name=name, version=version))
528 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
529 if name in files_in_apk:
531 files_in_apk.add(name)
536 def insert_obbs(repodir, apps, apks):
537 """Scans the .obb files in a given repo directory and adds them to the
538 relevant APK instances. OBB files have versionCodes like APK
539 files, and they are loosely associated. If there is an OBB file
540 present, then any APK with the same or higher versionCode will use
541 that OBB file. There are two OBB types: main and patch, each APK
542 can only have only have one of each.
544 https://developer.android.com/google/play/expansion-files.html
546 :param repodir: repo directory to scan
547 :param apps: list of current, valid apps
548 :param apks: current information on all APKs
552 def obbWarnDelete(f, msg):
553 logging.warning(msg + ' ' + f)
554 if options.delete_unknown:
555 logging.error(_("Deleting unknown file: {path}").format(path=f))
559 java_Integer_MIN_VALUE = -pow(2, 31)
560 currentPackageNames = apps.keys()
561 for f in glob.glob(os.path.join(repodir, '*.obb')):
562 obbfile = os.path.basename(f)
563 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
564 chunks = obbfile.split('.')
565 if chunks[0] != 'main' and chunks[0] != 'patch':
566 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
568 if not re.match(r'^-?[0-9]+$', chunks[1]):
569 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
570 .format(name=chunks[0]))
572 versionCode = int(chunks[1])
573 packagename = ".".join(chunks[2:-1])
575 highestVersionCode = java_Integer_MIN_VALUE
576 if packagename not in currentPackageNames:
577 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
580 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
581 highestVersionCode = apk['versionCode']
582 if versionCode > highestVersionCode:
583 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
584 .format(integer=str(versionCode)))
586 obbsha256 = sha256sum(f)
587 obbs.append((packagename, versionCode, obbfile, obbsha256))
590 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
591 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
592 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
593 apk['obbMainFile'] = obbfile
594 apk['obbMainFileSha256'] = obbsha256
595 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
596 apk['obbPatchFile'] = obbfile
597 apk['obbPatchFileSha256'] = obbsha256
598 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
602 def translate_per_build_anti_features(apps, apks):
603 """Grab the anti-features list from the build metadata
605 For most Anti-Features, they are really most applicable per-APK,
606 not for an app. An app can fix a vulnerability, add/remove
607 tracking, etc. This reads the 'antifeatures' list from the Build
608 entries in the fdroiddata metadata file, then transforms it into
609 the 'antiFeatures' list of unique items for the index.
611 The field key is all lower case in the metadata file to match the
612 rest of the Build fields. It is 'antiFeatures' camel case in the
613 implementation, index, and fdroidclient since it is translated
614 from the build 'antifeatures' field, not directly included.
618 antiFeatures = dict()
619 for packageName, app in apps.items():
621 for build in app['builds']:
622 afl = build.get('antifeatures')
624 d[int(build.versionCode)] = afl
626 antiFeatures[packageName] = d
629 d = antiFeatures.get(apk['packageName'])
631 afl = d.get(apk['versionCode'])
633 apk['antiFeatures'].update(afl)
636 def _get_localized_dict(app, locale):
637 '''get the dict to add localized store metadata to'''
638 if 'localized' not in app:
639 app['localized'] = collections.OrderedDict()
640 if locale not in app['localized']:
641 app['localized'][locale] = collections.OrderedDict()
642 return app['localized'][locale]
645 def _set_localized_text_entry(app, locale, key, f):
646 limit = config['char_limits'][key]
647 localized = _get_localized_dict(app, locale)
649 text = fp.read()[:limit]
651 localized[key] = text
654 def _set_author_entry(app, key, f):
655 limit = config['char_limits']['author']
657 text = fp.read()[:limit]
662 def copy_triple_t_store_metadata(apps):
663 """Include store metadata from the app's source repo
665 The Triple-T Gradle Play Publisher is a plugin that has a standard
666 file layout for all of the metadata and graphics that the Google
667 Play Store accepts. Since F-Droid has the git repo, it can just
668 pluck those files directly. This method reads any text files into
669 the app dict, then copies any graphics into the fdroid repo
672 This needs to be run before insert_localized_app_metadata() so that
673 the graphics files that are copied into the fdroid repo get
676 https://github.com/Triple-T/gradle-play-publisher#upload-images
677 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
681 if not os.path.isdir('build'):
682 return # nothing to do
684 for packageName, app in apps.items():
685 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
686 logging.debug('Triple-T Gradle Play Publisher: ' + d)
687 for root, dirs, files in os.walk(d):
688 segments = root.split('/')
689 locale = segments[-2]
691 if f == 'fulldescription':
692 _set_localized_text_entry(app, locale, 'description',
693 os.path.join(root, f))
695 elif f == 'shortdescription':
696 _set_localized_text_entry(app, locale, 'summary',
697 os.path.join(root, f))
700 _set_localized_text_entry(app, locale, 'name',
701 os.path.join(root, f))
704 _set_localized_text_entry(app, locale, 'video',
705 os.path.join(root, f))
707 elif f == 'whatsnew':
708 _set_localized_text_entry(app, segments[-1], 'whatsNew',
709 os.path.join(root, f))
711 elif f == 'contactEmail':
712 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
714 elif f == 'contactPhone':
715 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
717 elif f == 'contactWebsite':
718 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
721 base, extension = common.get_extension(f)
722 dirname = os.path.basename(root)
723 if extension in ALLOWED_EXTENSIONS \
724 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
725 if segments[-2] == 'listing':
726 locale = segments[-3]
728 locale = segments[-2]
729 destdir = os.path.join('repo', packageName, locale, dirname)
730 os.makedirs(destdir, mode=0o755, exist_ok=True)
731 sourcefile = os.path.join(root, f)
732 destfile = os.path.join(destdir, os.path.basename(f))
733 logging.debug('copying ' + sourcefile + ' ' + destfile)
734 shutil.copy(sourcefile, destfile)
737 def insert_localized_app_metadata(apps):
738 """scans standard locations for graphics and localized text
740 Scans for localized description files, store graphics, and
741 screenshot PNG files in statically defined screenshots directory
742 and adds them to the app metadata. The screenshots and graphic
743 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
744 and must be in the following layout:
745 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
747 repo/packageName/locale/featureGraphic.png
748 repo/packageName/locale/phoneScreenshots/1.png
749 repo/packageName/locale/phoneScreenshots/2.png
751 The changelog files must be text files named with the versionCode
752 ending with ".txt" and must be in the following layout:
753 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
755 repo/packageName/locale/changelogs/12345.txt
757 This will scan the each app's source repo then the metadata/ dir
758 for these standard locations of changelog files. If it finds
759 them, they will be added to the dict of all packages, with the
760 versions in the metadata/ folder taking precendence over the what
761 is in the app's source repo.
763 Where "packageName" is the app's packageName and "locale" is the locale
764 of the graphics, e.g. what language they are in, using the IETF RFC5646
765 format (en-US, fr-CA, es-MX, etc).
767 This will also scan the app's git for a fastlane folder, and the
768 metadata/ folder and the apps' source repos for standard locations
769 of graphic and screenshot files. If it finds them, it will copy
770 them into the repo. The fastlane files follow this pattern:
771 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
775 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
776 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
777 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
779 for srcd in sorted(sourcedirs):
780 if not os.path.isdir(srcd):
782 for root, dirs, files in os.walk(srcd):
783 segments = root.split('/')
784 packageName = segments[1]
785 if packageName not in apps:
786 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
788 locale = segments[-1]
789 destdir = os.path.join('repo', packageName, locale)
791 if f in ('description.txt', 'full_description.txt'):
792 _set_localized_text_entry(apps[packageName], locale, 'description',
793 os.path.join(root, f))
795 elif f in ('summary.txt', 'short_description.txt'):
796 _set_localized_text_entry(apps[packageName], locale, 'summary',
797 os.path.join(root, f))
799 elif f in ('name.txt', 'title.txt'):
800 _set_localized_text_entry(apps[packageName], locale, 'name',
801 os.path.join(root, f))
803 elif f == 'video.txt':
804 _set_localized_text_entry(apps[packageName], locale, 'video',
805 os.path.join(root, f))
807 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
808 locale = segments[-2]
809 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
810 os.path.join(root, f))
813 base, extension = common.get_extension(f)
814 if locale == 'images':
815 locale = segments[-2]
816 destdir = os.path.join('repo', packageName, locale)
817 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
818 os.makedirs(destdir, mode=0o755, exist_ok=True)
819 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
820 shutil.copy(os.path.join(root, f), destdir)
822 if d in SCREENSHOT_DIRS:
823 if locale == 'images':
824 locale = segments[-2]
825 destdir = os.path.join('repo', packageName, locale)
826 for f in glob.glob(os.path.join(root, d, '*.*')):
827 _ignored, extension = common.get_extension(f)
828 if extension in ALLOWED_EXTENSIONS:
829 screenshotdestdir = os.path.join(destdir, d)
830 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
831 logging.debug('copying ' + f + ' ' + screenshotdestdir)
832 shutil.copy(f, screenshotdestdir)
834 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
836 if not os.path.isdir(d):
838 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
839 if not os.path.isfile(f):
841 segments = f.split('/')
842 packageName = segments[1]
844 screenshotdir = segments[3]
845 filename = os.path.basename(f)
846 base, extension = common.get_extension(filename)
848 if packageName not in apps:
849 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
850 .format(path=filename, name=packageName))
852 graphics = _get_localized_dict(apps[packageName], locale)
854 if extension not in ALLOWED_EXTENSIONS:
855 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
856 elif base in GRAPHIC_NAMES:
857 # there can only be zero or one of these per locale
858 graphics[base] = filename
859 elif screenshotdir in SCREENSHOT_DIRS:
860 # there can any number of these per locale
861 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
862 if screenshotdir not in graphics:
863 graphics[screenshotdir] = []
864 graphics[screenshotdir].append(filename)
866 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
869 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
870 """Scan a repo for all files with an extension except APK/OBB
872 :param apkcache: current cached info about all repo files
873 :param repodir: repo directory to scan
874 :param knownapks: list of all known files, as per metadata.read_metadata
875 :param use_date_from_file: use date from file (instead of current date)
876 for newly added files
881 repodir = repodir.encode('utf-8')
882 for name in os.listdir(repodir):
883 file_extension = common.get_file_extension(name)
884 if file_extension == 'apk' or file_extension == 'obb':
886 filename = os.path.join(repodir, name)
887 name_utf8 = name.decode('utf-8')
888 if filename.endswith(b'_src.tar.gz'):
889 logging.debug(_('skipping source tarball: {path}')
890 .format(path=filename.decode('utf-8')))
892 if not common.is_repo_file(filename):
894 stat = os.stat(filename)
895 if stat.st_size == 0:
896 raise FDroidException(_('{path} is zero size!')
897 .format(path=filename))
899 shasum = sha256sum(filename)
902 repo_file = apkcache[name]
903 # added time is cached as tuple but used here as datetime instance
904 if 'added' in repo_file:
905 a = repo_file['added']
906 if isinstance(a, datetime):
907 repo_file['added'] = a
909 repo_file['added'] = datetime(*a[:6])
910 if repo_file.get('hash') == shasum:
911 logging.debug(_("Reading {apkfilename} from cache")
912 .format(apkfilename=name_utf8))
915 logging.debug(_("Ignoring stale cache data for {apkfilename}")
916 .format(apkfilename=name_utf8))
919 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
920 repo_file = collections.OrderedDict()
921 repo_file['name'] = os.path.splitext(name_utf8)[0]
922 # TODO rename apkname globally to something more generic
923 repo_file['apkName'] = name_utf8
924 repo_file['hash'] = shasum
925 repo_file['hashType'] = 'sha256'
926 repo_file['versionCode'] = 0
927 repo_file['versionName'] = shasum
928 # the static ID is the SHA256 unless it is set in the metadata
929 repo_file['packageName'] = shasum
931 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
933 repo_file['packageName'] = m.group(1)
934 repo_file['versionCode'] = int(m.group(2))
935 srcfilename = name + b'_src.tar.gz'
936 if os.path.exists(os.path.join(repodir, srcfilename)):
937 repo_file['srcname'] = srcfilename.decode('utf-8')
938 repo_file['size'] = stat.st_size
940 apkcache[name] = repo_file
943 if use_date_from_file:
944 timestamp = stat.st_ctime
945 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
947 default_date_param = None
949 # Record in knownapks, getting the added date at the same time..
950 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
951 default_date=default_date_param)
953 repo_file['added'] = added
955 repo_files.append(repo_file)
957 return repo_files, cachechanged
960 def scan_apk(apk_file):
962 Scans an APK file and returns dictionary with metadata of the APK.
964 Attention: This does *not* verify that the APK signature is correct.
966 :param apk_file: The (ideally absolute) path to the APK file
967 :raises BuildException
968 :return A dict containing APK metadata
971 'hash': sha256sum(apk_file),
972 'hashType': 'sha256',
973 'uses-permission': [],
974 'uses-permission-sdk-23': [],
978 'antiFeatures': set(),
981 if SdkToolsPopen(['aapt', 'version'], output=False):
982 scan_apk_aapt(apk, apk_file)
984 scan_apk_androguard(apk, apk_file)
986 # Get the signature, or rather the signing key fingerprints
987 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
988 apk['sig'] = getsig(apk_file)
990 raise BuildException("Failed to get apk signature")
991 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
993 if not apk.get('signer'):
994 raise BuildException("Failed to get apk signing key fingerprint")
996 # Get size of the APK
997 apk['size'] = os.path.getsize(apk_file)
999 if 'minSdkVersion' not in apk:
1000 logging.warning("No SDK version information found in {0}".format(apk_file))
1001 apk['minSdkVersion'] = 1
1003 # Check for known vulnerabilities
1004 if has_known_vulnerability(apk_file):
1005 apk['antiFeatures'].add('KnownVuln')
1010 def scan_apk_aapt(apk, apkfile):
1011 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1012 if p.returncode != 0:
1013 if options.delete_unknown:
1014 if os.path.exists(apkfile):
1015 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1018 logging.error("Could not find {0} to remove it".format(apkfile))
1020 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1021 raise BuildException(_("Invalid APK"))
1022 for line in p.output.splitlines():
1023 if line.startswith("package:"):
1025 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1026 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1027 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1028 except Exception as e:
1029 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1030 elif line.startswith("application:"):
1031 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1032 # Keep path to non-dpi icon in case we need it
1033 match = re.match(APK_ICON_PAT_NODPI, line)
1035 apk['icons_src']['-1'] = match.group(1)
1036 elif line.startswith("launchable-activity:"):
1037 # Only use launchable-activity as fallback to application
1039 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1040 if '-1' not in apk['icons_src']:
1041 match = re.match(APK_ICON_PAT_NODPI, line)
1043 apk['icons_src']['-1'] = match.group(1)
1044 elif line.startswith("application-icon-"):
1045 match = re.match(APK_ICON_PAT, line)
1047 density = match.group(1)
1048 path = match.group(2)
1049 apk['icons_src'][density] = path
1050 elif line.startswith("sdkVersion:"):
1051 m = re.match(APK_SDK_VERSION_PAT, line)
1053 logging.error(line.replace('sdkVersion:', '')
1054 + ' is not a valid minSdkVersion!')
1056 apk['minSdkVersion'] = m.group(1)
1057 # if target not set, default to min
1058 if 'targetSdkVersion' not in apk:
1059 apk['targetSdkVersion'] = m.group(1)
1060 elif line.startswith("targetSdkVersion:"):
1061 m = re.match(APK_SDK_VERSION_PAT, line)
1063 logging.error(line.replace('targetSdkVersion:', '')
1064 + ' is not a valid targetSdkVersion!')
1066 apk['targetSdkVersion'] = m.group(1)
1067 elif line.startswith("maxSdkVersion:"):
1068 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1069 elif line.startswith("native-code:"):
1070 apk['nativecode'] = []
1071 for arch in line[13:].split(' '):
1072 apk['nativecode'].append(arch[1:-1])
1073 elif line.startswith('uses-permission:'):
1074 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1075 if perm_match['maxSdkVersion']:
1076 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1077 permission = UsesPermission(
1079 perm_match['maxSdkVersion']
1082 apk['uses-permission'].append(permission)
1083 elif line.startswith('uses-permission-sdk-23:'):
1084 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1085 if perm_match['maxSdkVersion']:
1086 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1087 permission_sdk_23 = UsesPermissionSdk23(
1089 perm_match['maxSdkVersion']
1092 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1094 elif line.startswith('uses-feature:'):
1095 feature = re.match(APK_FEATURE_PAT, line).group(1)
1096 # Filter out this, it's only added with the latest SDK tools and
1097 # causes problems for lots of apps.
1098 if feature != "android.hardware.screen.portrait" \
1099 and feature != "android.hardware.screen.landscape":
1100 if feature.startswith("android.feature."):
1101 feature = feature[16:]
1102 apk['features'].add(feature)
1105 def scan_apk_androguard(apk, apkfile):
1107 from androguard.core.bytecodes.apk import APK
1108 apkobject = APK(apkfile)
1109 if apkobject.is_valid_APK():
1110 arsc = apkobject.get_android_resources()
1112 if options.delete_unknown:
1113 if os.path.exists(apkfile):
1114 logging.error(_("Failed to get apk information, deleting {path}")
1115 .format(path=apkfile))
1118 logging.error(_("Could not find {path} to remove it")
1119 .format(path=apkfile))
1121 logging.error(_("Failed to get apk information, skipping {path}")
1122 .format(path=apkfile))
1123 raise BuildException(_("Invalid APK"))
1125 raise FDroidException("androguard library is not installed and aapt not present")
1126 except FileNotFoundError:
1127 logging.error(_("Could not open apk file for analysis"))
1128 raise BuildException(_("Invalid APK"))
1130 apk['packageName'] = apkobject.get_package()
1131 apk['versionCode'] = int(apkobject.get_androidversion_code())
1132 apk['versionName'] = apkobject.get_androidversion_name()
1133 if apk['versionName'][0] == "@":
1134 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1135 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1136 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1137 apk['name'] = apkobject.get_app_name()
1139 if apkobject.get_max_sdk_version() is not None:
1140 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1141 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1142 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1144 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1145 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1147 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1149 for file in apkobject.get_files():
1150 d_re = density_re.match(file)
1152 folder = d_re.group(1).split('-')
1154 resolution = folder[1]
1157 density = screen_resolutions[resolution]
1158 apk['icons_src'][density] = d_re.group(0)
1160 if apk['icons_src'].get('-1') is None:
1161 apk['icons_src']['-1'] = apk['icons_src']['160']
1163 arch_re = re.compile("^lib/(.*)/.*$")
1164 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1166 apk['nativecode'] = []
1167 apk['nativecode'].extend(sorted(list(arch)))
1169 xml = apkobject.get_android_manifest_xml()
1171 for item in xml.getElementsByTagName('uses-permission'):
1172 name = str(item.getAttribute("android:name"))
1173 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1174 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1175 permission = UsesPermission(
1179 apk['uses-permission'].append(permission)
1181 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1182 name = str(item.getAttribute("android:name"))
1183 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1184 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1185 permission_sdk_23 = UsesPermissionSdk23(
1189 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1191 for item in xml.getElementsByTagName('uses-feature'):
1192 feature = str(item.getAttribute("android:name"))
1193 if feature != "android.hardware.screen.portrait" \
1194 and feature != "android.hardware.screen.landscape":
1195 if feature.startswith("android.feature."):
1196 feature = feature[16:]
1197 apk['features'].append(feature)
1200 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1201 allow_disabled_algorithms=False, archive_bad_sig=False):
1202 """Processes the apk with the given filename in the given repo directory.
1204 This also extracts the icons.
1206 :param apkcache: current apk cache information
1207 :param apkfilename: the filename of the apk to scan
1208 :param repodir: repo directory to scan
1209 :param knownapks: known apks info
1210 :param use_date_from_apk: use date from APK (instead of current date)
1211 for newly added APKs
1212 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1213 disabled algorithms in the signature (e.g. MD5)
1214 :param archive_bad_sig: move APKs with a bad signature to the archive
1215 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1216 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1220 apkfile = os.path.join(repodir, apkfilename)
1222 cachechanged = False
1224 if apkfilename in apkcache:
1225 apk = apkcache[apkfilename]
1226 if apk.get('hash') == sha256sum(apkfile):
1227 logging.debug(_("Reading {apkfilename} from cache")
1228 .format(apkfilename=apkfilename))
1231 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1232 .format(apkfilename=apkfilename))
1235 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1238 apk = scan_apk(apkfile)
1239 except BuildException:
1240 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1241 .format(apkfilename=apkfilename))
1242 return True, None, False
1244 # Check for debuggable apks...
1245 if common.isApkAndDebuggable(apkfile):
1246 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1248 if options.rename_apks:
1249 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1250 std_short_name = os.path.join(repodir, n)
1251 if apkfile != std_short_name:
1252 if os.path.exists(std_short_name):
1253 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1254 if apkfile != std_long_name:
1255 if os.path.exists(std_long_name):
1256 dupdir = os.path.join('duplicates', repodir)
1257 if not os.path.isdir(dupdir):
1258 os.makedirs(dupdir, exist_ok=True)
1259 dupfile = os.path.join('duplicates', std_long_name)
1260 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1261 os.rename(apkfile, dupfile)
1262 return True, None, False
1264 os.rename(apkfile, std_long_name)
1265 apkfile = std_long_name
1267 os.rename(apkfile, std_short_name)
1268 apkfile = std_short_name
1269 apkfilename = apkfile[len(repodir) + 1:]
1271 apk['apkName'] = apkfilename
1272 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1273 if os.path.exists(os.path.join(repodir, srcfilename)):
1274 apk['srcname'] = srcfilename
1276 # verify the jar signature is correct, allow deprecated
1277 # algorithms only if the APK is in the archive.
1279 if not common.verify_apk_signature(apkfile):
1280 if repodir == 'archive' or allow_disabled_algorithms:
1281 if common.verify_old_apk_signature(apkfile):
1282 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1290 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1291 .format(apkfilename=apkfilename))
1292 move_apk_between_sections(repodir, 'archive', apk)
1294 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1295 .format(apkfilename=apkfilename))
1296 return True, None, False
1298 apkzip = zipfile.ZipFile(apkfile, 'r')
1300 # if an APK has files newer than the system time, suggest updating
1301 # the system clock. This is useful for offline systems, used for
1302 # signing, which do not have another source of clock sync info. It
1303 # has to be more than 24 hours newer because ZIP/APK files do not
1304 # store timezone info
1305 manifest = apkzip.getinfo('AndroidManifest.xml')
1306 if manifest.date_time[1] == 0: # month can't be zero
1307 logging.debug(_('AndroidManifest.xml has no date'))
1309 dt_obj = datetime(*manifest.date_time)
1310 checkdt = dt_obj - timedelta(1)
1311 if datetime.today() < checkdt:
1312 logging.warning('System clock is older than manifest in: '
1314 + '\nSet clock to that time using:\n'
1315 + 'sudo date -s "' + str(dt_obj) + '"')
1317 # extract icons from APK zip file
1318 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1320 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1322 apkzip.close() # ensure that APK zip file gets closed
1324 # resize existing icons for densities missing in the APK
1325 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1327 if use_date_from_apk and manifest.date_time[1] != 0:
1328 default_date_param = datetime(*manifest.date_time)
1330 default_date_param = None
1332 # Record in known apks, getting the added date at the same time..
1333 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1334 default_date=default_date_param)
1336 apk['added'] = added
1338 apkcache[apkfilename] = apk
1341 return False, apk, cachechanged
1344 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1345 """Processes the apks in the given repo directory.
1347 This also extracts the icons.
1349 :param apkcache: current apk cache information
1350 :param repodir: repo directory to scan
1351 :param knownapks: known apks info
1352 :param use_date_from_apk: use date from APK (instead of current date)
1353 for newly added APKs
1354 :returns: (apks, cachechanged) where apks is a list of apk information,
1355 and cachechanged is True if the apkcache got changed.
1358 cachechanged = False
1360 for icon_dir in get_all_icon_dirs(repodir):
1361 if os.path.exists(icon_dir):
1363 shutil.rmtree(icon_dir)
1364 os.makedirs(icon_dir)
1366 os.makedirs(icon_dir)
1369 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1370 apkfilename = apkfile[len(repodir) + 1:]
1371 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1372 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1373 use_date_from_apk, ada, True)
1377 cachechanged = cachechanged or cachethis
1379 return apks, cachechanged
1382 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1384 Extracts icons from the given APK zip in various densities,
1385 saves them into given repo directory
1386 and stores their names in the APK metadata dictionary.
1388 :param icon_filename: A string representing the icon's file name
1389 :param apk: A populated dictionary containing APK metadata.
1390 Needs to have 'icons_src' key
1391 :param apkzip: An opened zipfile.ZipFile of the APK file
1392 :param repo_dir: The directory of the APK's repository
1393 :return: A list of icon densities that are missing
1395 empty_densities = []
1396 for density in screen_densities:
1397 if density not in apk['icons_src']:
1398 empty_densities.append(density)
1400 icon_src = apk['icons_src'][density]
1401 icon_dir = get_icon_dir(repo_dir, density)
1402 icon_dest = os.path.join(icon_dir, icon_filename)
1404 # Extract the icon files per density
1405 if icon_src.endswith('.xml'):
1406 png = os.path.basename(icon_src)[:-4] + '.png'
1407 for f in apkzip.namelist():
1409 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1410 if m and screen_resolutions[m.group(2)] == density:
1412 if icon_src.endswith('.xml'):
1413 empty_densities.append(density)
1416 with open(icon_dest, 'wb') as f:
1417 f.write(get_icon_bytes(apkzip, icon_src))
1418 apk['icons'][density] = icon_filename
1419 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1420 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1421 del apk['icons_src'][density]
1422 empty_densities.append(density)
1424 if '-1' in apk['icons_src']:
1425 icon_src = apk['icons_src']['-1']
1426 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1427 with open(icon_path, 'wb') as f:
1428 f.write(get_icon_bytes(apkzip, icon_src))
1430 im = Image.open(icon_path)
1431 dpi = px_to_dpi(im.size[0])
1432 for density in screen_densities:
1433 if density in apk['icons']:
1435 if density == screen_densities[-1] or dpi >= int(density):
1436 apk['icons'][density] = icon_filename
1437 shutil.move(icon_path,
1438 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1439 empty_densities.remove(density)
1441 except Exception as e:
1442 logging.warning(_("Failed reading {path}: {error}")
1443 .format(path=icon_path, error=e))
1446 apk['icon'] = icon_filename
1448 return empty_densities
1451 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1453 Resize existing icons for densities missing in the APK to ensure all densities are available
1455 :param empty_densities: A list of icon densities that are missing
1456 :param icon_filename: A string representing the icon's file name
1457 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1458 :param repo_dir: The directory of the APK's repository
1460 # First try resizing down to not lose quality
1462 for density in screen_densities:
1463 if density not in empty_densities:
1464 last_density = density
1466 if last_density is None:
1468 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1470 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1471 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1474 fp = open(last_icon_path, 'rb')
1477 size = dpi_to_px(density)
1479 im.thumbnail((size, size), Image.ANTIALIAS)
1480 im.save(icon_path, "PNG")
1481 empty_densities.remove(density)
1482 except Exception as e:
1483 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1488 # Then just copy from the highest resolution available
1490 for density in reversed(screen_densities):
1491 if density not in empty_densities:
1492 last_density = density
1495 if last_density is None:
1499 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1500 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1502 empty_densities.remove(density)
1504 for density in screen_densities:
1505 icon_dir = get_icon_dir(repo_dir, density)
1506 icon_dest = os.path.join(icon_dir, icon_filename)
1507 resize_icon(icon_dest, density)
1509 # Copy from icons-mdpi to icons since mdpi is the baseline density
1510 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1511 if os.path.isfile(baseline):
1512 apk['icons']['0'] = icon_filename
1513 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1516 def apply_info_from_latest_apk(apps, apks):
1518 Some information from the apks needs to be applied up to the application level.
1519 When doing this, we use the info from the most recent version's apk.
1520 We deal with figuring out when the app was added and last updated at the same time.
1522 for appid, app in apps.items():
1523 bestver = UNSET_VERSION_CODE
1525 if apk['packageName'] == appid:
1526 if apk['versionCode'] > bestver:
1527 bestver = apk['versionCode']
1531 if not app.added or apk['added'] < app.added:
1532 app.added = apk['added']
1533 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1534 app.lastUpdated = apk['added']
1537 logging.debug("Don't know when " + appid + " was added")
1538 if not app.lastUpdated:
1539 logging.debug("Don't know when " + appid + " was last updated")
1541 if bestver == UNSET_VERSION_CODE:
1543 if app.Name is None:
1544 app.Name = app.AutoName or appid
1546 logging.debug("Application " + appid + " has no packages")
1548 if app.Name is None:
1549 app.Name = bestapk['name']
1550 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1551 if app.CurrentVersionCode is None:
1552 app.CurrentVersionCode = str(bestver)
1555 def make_categories_txt(repodir, categories):
1556 '''Write a category list in the repo to allow quick access'''
1558 for cat in sorted(categories):
1559 catdata += cat + '\n'
1560 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1564 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1566 def filter_apk_list_sorted(apk_list):
1568 for apk in apk_list:
1569 if apk['packageName'] == appid:
1572 # Sort the apk list by version code. First is highest/newest.
1573 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1575 for appid, app in apps.items():
1577 if app.ArchivePolicy:
1578 keepversions = int(app.ArchivePolicy[:-9])
1580 keepversions = defaultkeepversions
1582 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1583 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1585 current_app_apks = filter_apk_list_sorted(apks)
1586 if len(current_app_apks) > keepversions:
1587 # Move back the ones we don't want.
1588 for apk in current_app_apks[keepversions:]:
1589 move_apk_between_sections(repodir, archivedir, apk)
1590 archapks.append(apk)
1593 current_app_archapks = filter_apk_list_sorted(archapks)
1594 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1596 # Move forward the ones we want again, except DisableAlgorithm
1597 for apk in current_app_archapks:
1598 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1599 move_apk_between_sections(archivedir, repodir, apk)
1600 archapks.remove(apk)
1603 if kept == keepversions:
1607 def move_apk_between_sections(from_dir, to_dir, apk):
1608 """move an APK from repo to archive or vice versa"""
1610 def _move_file(from_dir, to_dir, filename, ignore_missing):
1611 from_path = os.path.join(from_dir, filename)
1612 if ignore_missing and not os.path.exists(from_path):
1614 to_path = os.path.join(to_dir, filename)
1615 if not os.path.exists(to_dir):
1617 shutil.move(from_path, to_path)
1619 if from_dir == to_dir:
1622 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1623 _move_file(from_dir, to_dir, apk['apkName'], False)
1624 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1625 for density in all_screen_densities:
1626 from_icon_dir = get_icon_dir(from_dir, density)
1627 to_icon_dir = get_icon_dir(to_dir, density)
1628 if density not in apk.get('icons', []):
1630 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1631 if 'srcname' in apk:
1632 _move_file(from_dir, to_dir, apk['srcname'], False)
1635 def add_apks_to_per_app_repos(repodir, apks):
1636 apks_per_app = dict()
1638 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1639 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1640 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1641 apks_per_app[apk['packageName']] = apk
1643 if not os.path.exists(apk['per_app_icons']):
1644 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1645 os.makedirs(apk['per_app_icons'])
1647 apkpath = os.path.join(repodir, apk['apkName'])
1648 shutil.copy(apkpath, apk['per_app_repo'])
1649 apksigpath = apkpath + '.sig'
1650 if os.path.exists(apksigpath):
1651 shutil.copy(apksigpath, apk['per_app_repo'])
1652 apkascpath = apkpath + '.asc'
1653 if os.path.exists(apkascpath):
1654 shutil.copy(apkascpath, apk['per_app_repo'])
1657 def create_metadata_from_template(apk):
1658 '''create a new metadata file using internal or external template
1660 Generate warnings for apk's with no metadata (or create skeleton
1661 metadata files, if requested on the command line). Though the
1662 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1663 since those impose things on the metadata file made from the
1664 template: field sort order, empty field value, formatting, etc.
1668 if os.path.exists('template.yml'):
1669 with open('template.yml') as f:
1671 if 'name' in apk and apk['name'] != '':
1672 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1673 r'\1 ' + apk['name'],
1675 flags=re.IGNORECASE | re.MULTILINE)
1677 logging.warning(_('{appid} does not have a name! Using package name instead.')
1678 .format(appid=apk['packageName']))
1679 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1680 r'\1 ' + apk['packageName'],
1682 flags=re.IGNORECASE | re.MULTILINE)
1683 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1687 app['Categories'] = [os.path.basename(os.getcwd())]
1688 # include some blanks as part of the template
1689 app['AuthorName'] = ''
1692 app['IssueTracker'] = ''
1693 app['SourceCode'] = ''
1694 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1695 if 'name' in apk and apk['name'] != '':
1696 app['Name'] = apk['name']
1698 logging.warning(_('{appid} does not have a name! Using package name instead.')
1699 .format(appid=apk['packageName']))
1700 app['Name'] = apk['packageName']
1701 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1702 yaml.dump(app, f, default_flow_style=False)
1703 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1712 global config, options
1714 # Parse command line...
1715 parser = ArgumentParser()
1716 common.setup_global_opts(parser)
1717 parser.add_argument("--create-key", action="store_true", default=False,
1718 help=_("Add a repo signing key to an unsigned repo"))
1719 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1720 help=_("Add skeleton metadata files for APKs that are missing them"))
1721 parser.add_argument("--delete-unknown", action="store_true", default=False,
1722 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1723 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1724 help=_("Report on build data status"))
1725 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1726 help=_("Interactively ask about things that need updating."))
1727 parser.add_argument("-I", "--icons", action="store_true", default=False,
1728 help=_("Resize all the icons exceeding the max pixel size and exit"))
1729 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1730 help=_("Specify editor to use in interactive mode. Default " +
1731 "is {path}").format(path='/etc/alternatives/editor'))
1732 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1733 help=_("Update the wiki"))
1734 parser.add_argument("--pretty", action="store_true", default=False,
1735 help=_("Produce human-readable XML/JSON for index files"))
1736 parser.add_argument("--clean", action="store_true", default=False,
1737 help=_("Clean update - don't uses caches, reprocess all APKs"))
1738 parser.add_argument("--nosign", action="store_true", default=False,
1739 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1740 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1741 help=_("Use date from APK instead of current time for newly added APKs"))
1742 parser.add_argument("--rename-apks", action="store_true", default=False,
1743 help=_("Rename APK files that do not match package.name_123.apk"))
1744 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1745 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1746 metadata.add_metadata_arguments(parser)
1747 options = parser.parse_args()
1748 metadata.warnings_action = options.W
1750 config = common.read_config(options)
1752 if not ('jarsigner' in config and 'keytool' in config):
1753 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1756 if config['archive_older'] != 0:
1757 repodirs.append('archive')
1758 if not os.path.exists('archive'):
1762 resize_all_icons(repodirs)
1765 if options.rename_apks:
1766 options.clean = True
1768 # check that icons exist now, rather than fail at the end of `fdroid update`
1769 for k in ['repo_icon', 'archive_icon']:
1771 if not os.path.exists(config[k]):
1772 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1773 .format(name=k, path=config[k]))
1776 # if the user asks to create a keystore, do it now, reusing whatever it can
1777 if options.create_key:
1778 if os.path.exists(config['keystore']):
1779 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1780 logging.critical("\t'" + config['keystore'] + "'")
1783 if 'repo_keyalias' not in config:
1784 config['repo_keyalias'] = socket.getfqdn()
1785 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1786 if 'keydname' not in config:
1787 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1788 common.write_to_config(config, 'keydname', config['keydname'])
1789 if 'keystore' not in config:
1790 config['keystore'] = common.default_config['keystore']
1791 common.write_to_config(config, 'keystore', config['keystore'])
1793 password = common.genpassword()
1794 if 'keystorepass' not in config:
1795 config['keystorepass'] = password
1796 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1797 if 'keypass' not in config:
1798 config['keypass'] = password
1799 common.write_to_config(config, 'keypass', config['keypass'])
1800 common.genkeystore(config)
1803 apps = metadata.read_metadata()
1805 # Generate a list of categories...
1807 for app in apps.values():
1808 categories.update(app.Categories)
1810 # Read known apks data (will be updated and written back when we've finished)
1811 knownapks = common.KnownApks()
1814 apkcache = get_cache()
1816 # Delete builds for disabled apps
1817 delete_disabled_builds(apps, apkcache, repodirs)
1819 # Scan all apks in the main repo
1820 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1822 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1823 options.use_date_from_apk)
1824 cachechanged = cachechanged or fcachechanged
1827 if apk['packageName'] not in apps:
1828 if options.create_metadata:
1829 create_metadata_from_template(apk)
1830 apps = metadata.read_metadata()
1832 msg = _("{apkfilename} ({appid}) has no metadata!") \
1833 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1834 if options.delete_unknown:
1835 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1836 .format(apkfilename=apk['apkName']))
1837 rmf = os.path.join(repodirs[0], apk['apkName'])
1838 if not os.path.exists(rmf):
1839 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1843 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1845 copy_triple_t_store_metadata(apps)
1846 insert_obbs(repodirs[0], apps, apks)
1847 insert_localized_app_metadata(apps)
1848 translate_per_build_anti_features(apps, apks)
1850 # Scan the archive repo for apks as well
1851 if len(repodirs) > 1:
1852 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1858 # Apply information from latest apks to the application and update dates
1859 apply_info_from_latest_apk(apps, apks + archapks)
1861 # Sort the app list by name, then the web site doesn't have to by default.
1862 # (we had to wait until we'd scanned the apks to do this, because mostly the
1863 # name comes from there!)
1864 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1866 # APKs are placed into multiple repos based on the app package, providing
1867 # per-app subscription feeds for nightly builds and things like it
1868 if config['per_app_repos']:
1869 add_apks_to_per_app_repos(repodirs[0], apks)
1870 for appid, app in apps.items():
1871 repodir = os.path.join(appid, 'fdroid', 'repo')
1873 appdict[appid] = app
1874 if os.path.isdir(repodir):
1875 index.make(appdict, [appid], apks, repodir, False)
1877 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1880 if len(repodirs) > 1:
1881 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1883 # Make the index for the main repo...
1884 index.make(apps, sortedids, apks, repodirs[0], False)
1885 make_categories_txt(repodirs[0], categories)
1887 # If there's an archive repo, make the index for it. We already scanned it
1889 if len(repodirs) > 1:
1890 index.make(apps, sortedids, archapks, repodirs[1], True)
1892 git_remote = config.get('binary_transparency_remote')
1893 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1895 btlog.make_binary_transparency_log(repodirs)
1897 if config['update_stats']:
1898 # Update known apks info...
1899 knownapks.writeifchanged()
1901 # Generate latest apps data for widget
1902 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1904 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1906 appid = line.rstrip()
1907 data += appid + "\t"
1909 data += app.Name + "\t"
1910 if app.icon is not None:
1911 data += app.icon + "\t"
1912 data += app.License + "\n"
1913 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1917 write_cache(apkcache)
1919 # Update the wiki...
1921 update_wiki(apps, sortedids, apks + archapks)
1923 logging.info(_("Finished"))
1926 if __name__ == "__main__":