3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
98 return os.path.join(repodir, "icons")
99 return os.path.join(repodir, "icons-%s" % density)
102 def get_icon_dirs(repodir):
103 for density in screen_densities:
104 yield get_icon_dir(repodir, density)
107 def get_all_icon_dirs(repodir):
108 for density in all_screen_densities:
109 yield get_icon_dir(repodir, density)
112 def update_wiki(apps, sortedids, apks):
115 :param apps: fully populated list of all applications
116 :param apks: all apks, except...
118 logging.info("Updating wiki")
120 wikiredircat = 'App Redirects'
122 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
123 path=config['wiki_path'])
124 site.login(config['wiki_user'], config['wiki_password'])
126 generated_redirects = {}
128 for appid in sortedids:
129 app = metadata.App(apps[appid])
133 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
135 for af in sorted(app.AntiFeatures):
136 wikidata += '{{AntiFeature|' + af + '}}\n'
141 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
144 app.added.strftime('%Y-%m-%d') if app.added else '',
145 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
161 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
163 wikidata += app.Summary
164 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
166 wikidata += "=Description=\n"
167 wikidata += metadata.description_wiki(app.Description) + "\n"
169 wikidata += "=Maintainer Notes=\n"
170 if app.MaintainerNotes:
171 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
172 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
174 # Get a list of all packages for this application...
176 gotcurrentver = False
180 if apk['packageName'] == appid:
181 if str(apk['versionCode']) == app.CurrentVersionCode:
184 # Include ones we can't build, as a special case...
185 for build in app.builds:
187 if build.versionCode == app.CurrentVersionCode:
189 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
190 apklist.append({'versionCode': int(build.versionCode),
191 'versionName': build.versionName,
192 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
197 if apk['versionCode'] == int(build.versionCode):
202 apklist.append({'versionCode': int(build.versionCode),
203 'versionName': build.versionName,
204 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
206 if app.CurrentVersionCode == '0':
208 # Sort with most recent first...
209 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
211 wikidata += "=Versions=\n"
212 if len(apklist) == 0:
213 wikidata += "We currently have no versions of this app available."
214 elif not gotcurrentver:
215 wikidata += "We don't have the current version of this app."
217 wikidata += "We have the current version of this app."
218 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
219 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
220 if len(app.NoSourceSince) > 0:
221 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
222 if len(app.CurrentVersion) > 0:
223 wikidata += "The current (recommended) version is " + app.CurrentVersion
224 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
227 wikidata += "==" + apk['versionName'] + "==\n"
229 if 'buildproblem' in apk:
230 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
233 wikidata += "This version is built and signed by "
235 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
237 wikidata += "the original developer.\n\n"
238 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
240 wikidata += '\n[[Category:' + wikicat + ']]\n'
241 if len(app.NoSourceSince) > 0:
242 wikidata += '\n[[Category:Apps missing source code]]\n'
243 if validapks == 0 and not app.Disabled:
244 wikidata += '\n[[Category:Apps with no packages]]\n'
245 if cantupdate and not app.Disabled:
246 wikidata += "\n[[Category:Apps we cannot update]]\n"
247 if buildfails and not app.Disabled:
248 wikidata += "\n[[Category:Apps with failing builds]]\n"
249 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
250 wikidata += '\n[[Category:Apps to Update]]\n'
252 wikidata += '\n[[Category:Apps that are disabled]]\n'
253 if app.UpdateCheckMode == 'None' and not app.Disabled:
254 wikidata += '\n[[Category:Apps with no update check]]\n'
255 for appcat in app.Categories:
256 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
258 # We can't have underscores in the page name, even if they're in
259 # the package ID, because MediaWiki messes with them...
260 pagename = appid.replace('_', ' ')
262 # Drop a trailing newline, because mediawiki is going to drop it anyway
263 # and it we don't we'll think the page has changed when it hasn't...
264 if wikidata.endswith('\n'):
265 wikidata = wikidata[:-1]
267 generated_pages[pagename] = wikidata
269 # Make a redirect from the name to the ID too, unless there's
270 # already an existing page with the name and it isn't a redirect.
272 apppagename = app.Name.replace('_', ' ')
273 apppagename = apppagename.replace('{', '')
274 apppagename = apppagename.replace('}', ' ')
275 apppagename = apppagename.replace(':', ' ')
276 apppagename = apppagename.replace('[', ' ')
277 apppagename = apppagename.replace(']', ' ')
278 # Drop double spaces caused mostly by replacing ':' above
279 apppagename = apppagename.replace(' ', ' ')
280 for expagename in site.allpages(prefix=apppagename,
281 filterredir='nonredirects',
283 if expagename == apppagename:
285 # Another reason not to make the redirect page is if the app name
286 # is the same as it's ID, because that will overwrite the real page
287 # with an redirect to itself! (Although it seems like an odd
288 # scenario this happens a lot, e.g. where there is metadata but no
289 # builds or binaries to extract a name from.
290 if apppagename == pagename:
293 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
295 for tcat, genp in [(wikicat, generated_pages),
296 (wikiredircat, generated_redirects)]:
297 catpages = site.Pages['Category:' + tcat]
299 for page in catpages:
300 existingpages.append(page.name)
301 if page.name in genp:
302 pagetxt = page.edit()
303 if pagetxt != genp[page.name]:
304 logging.debug("Updating modified page " + page.name)
305 page.save(genp[page.name], summary='Auto-updated')
307 logging.debug("Page " + page.name + " is unchanged")
309 logging.warn("Deleting page " + page.name)
310 page.delete('No longer published')
311 for pagename, text in genp.items():
312 logging.debug("Checking " + pagename)
313 if pagename not in existingpages:
314 logging.debug("Creating page " + pagename)
316 newpage = site.Pages[pagename]
317 newpage.save(text, summary='Auto-created')
318 except Exception as e:
319 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
321 # Purge server cache to ensure counts are up to date
322 site.pages['Repository Maintenance'].purge()
325 def delete_disabled_builds(apps, apkcache, repodirs):
326 """Delete disabled build outputs.
328 :param apps: list of all applications, as per metadata.read_metadata
329 :param apkcache: current apk cache information
330 :param repodirs: the repo directories to process
332 for appid, app in apps.items():
333 for build in app['builds']:
334 if not build.disable:
336 apkfilename = common.get_release_filename(app, build)
337 iconfilename = "%s.%s.png" % (
340 for repodir in repodirs:
342 os.path.join(repodir, apkfilename),
343 os.path.join(repodir, apkfilename + '.asc'),
344 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
346 for density in all_screen_densities:
347 repo_dir = get_icon_dir(repodir, density)
348 files.append(os.path.join(repo_dir, iconfilename))
351 if os.path.exists(f):
352 logging.info("Deleting disabled build output " + f)
354 if apkfilename in apkcache:
355 del apkcache[apkfilename]
358 def resize_icon(iconpath, density):
360 if not os.path.isfile(iconpath):
365 fp = open(iconpath, 'rb')
367 size = dpi_to_px(density)
369 if any(length > size for length in im.size):
371 im.thumbnail((size, size), Image.ANTIALIAS)
372 logging.debug("%s was too large at %s - new size is %s" % (
373 iconpath, oldsize, im.size))
374 im.save(iconpath, "PNG")
376 except Exception as e:
377 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
384 def resize_all_icons(repodirs):
385 """Resize all icons that exceed the max size
387 :param repodirs: the repo directories to process
389 for repodir in repodirs:
390 for density in screen_densities:
391 icon_dir = get_icon_dir(repodir, density)
392 icon_glob = os.path.join(icon_dir, '*.png')
393 for iconpath in glob.glob(icon_glob):
394 resize_icon(iconpath, density)
398 """ Get the signing certificate of an apk. To get the same md5 has that
399 Android gets, we encode the .RSA certificate in a specific format and pass
400 it hex-encoded to the md5 digest algorithm.
402 :param apkpath: path to the apk
403 :returns: A string containing the md5 of the signature of the apk or None
404 if an error occurred.
407 with zipfile.ZipFile(apkpath, 'r') as apk:
408 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
411 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
414 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
417 cert = apk.read(certs[0])
419 cert_encoded = common.get_certificate(cert)
421 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
424 def get_cache_file():
425 return os.path.join('tmp', 'apkcache')
429 """Get the cached dict of the APK index
431 Gather information about all the apk files in the repo directory,
432 using cached data if possible. Some of the index operations take a
433 long time, like calculating the SHA-256 and verifying the APK
436 The cache is invalidated if the metadata version is different, or
437 the 'allow_disabled_algorithms' config/option is different. In
438 those cases, there is no easy way to know what has changed from
439 the cache, so just rerun the whole thing.
444 apkcachefile = get_cache_file()
445 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
446 if not options.clean and os.path.exists(apkcachefile):
447 with open(apkcachefile, 'rb') as cf:
448 apkcache = pickle.load(cf, encoding='utf-8')
449 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
450 or apkcache.get('allow_disabled_algorithms') != ada:
455 apkcache["METADATA_VERSION"] = METADATA_VERSION
456 apkcache['allow_disabled_algorithms'] = ada
461 def write_cache(apkcache):
462 apkcachefile = get_cache_file()
463 cache_path = os.path.dirname(apkcachefile)
464 if not os.path.exists(cache_path):
465 os.makedirs(cache_path)
466 with open(apkcachefile, 'wb') as cf:
467 pickle.dump(apkcache, cf)
470 def get_icon_bytes(apkzip, iconsrc):
471 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
473 return apkzip.read(iconsrc)
475 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
478 def sha256sum(filename):
479 '''Calculate the sha256 of the given file'''
480 sha = hashlib.sha256()
481 with open(filename, 'rb') as f:
487 return sha.hexdigest()
490 def has_known_vulnerability(filename):
491 """checks for known vulnerabilities in the APK
493 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
494 version. Google also enforces this:
495 https://support.google.com/faqs/answer/6376725?hl=en
497 Checks whether there are more than one classes.dex or AndroidManifest.xml
498 files, which is invalid and an essential part of the "Master Key" attack.
500 http://www.saurik.com/id/17
505 # statically load this pattern
506 if not hasattr(has_known_vulnerability, "pattern"):
507 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
510 with zipfile.ZipFile(filename) as zf:
511 for name in zf.namelist():
512 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
515 chunk = lib.read(4096)
518 m = has_known_vulnerability.pattern.search(chunk)
520 version = m.group(1).decode('ascii')
521 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
522 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
523 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
524 logging.debug(_('"{path}" contains recent {name} ({version})')
525 .format(path=filename, name=name, version=version))
527 logging.warning(_('"{path}" contains outdated {name} ({version})')
528 .format(path=filename, name=name, version=version))
531 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
532 if name in files_in_apk:
533 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
534 .format(apkfilename=filename, name=name))
536 files_in_apk.add(name)
540 def insert_obbs(repodir, apps, apks):
541 """Scans the .obb files in a given repo directory and adds them to the
542 relevant APK instances. OBB files have versionCodes like APK
543 files, and they are loosely associated. If there is an OBB file
544 present, then any APK with the same or higher versionCode will use
545 that OBB file. There are two OBB types: main and patch, each APK
546 can only have only have one of each.
548 https://developer.android.com/google/play/expansion-files.html
550 :param repodir: repo directory to scan
551 :param apps: list of current, valid apps
552 :param apks: current information on all APKs
556 def obbWarnDelete(f, msg):
557 logging.warning(msg + ' ' + f)
558 if options.delete_unknown:
559 logging.error(_("Deleting unknown file: {path}").format(path=f))
563 java_Integer_MIN_VALUE = -pow(2, 31)
564 currentPackageNames = apps.keys()
565 for f in glob.glob(os.path.join(repodir, '*.obb')):
566 obbfile = os.path.basename(f)
567 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
568 chunks = obbfile.split('.')
569 if chunks[0] != 'main' and chunks[0] != 'patch':
570 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
572 if not re.match(r'^-?[0-9]+$', chunks[1]):
573 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
574 .format(name=chunks[0]))
576 versionCode = int(chunks[1])
577 packagename = ".".join(chunks[2:-1])
579 highestVersionCode = java_Integer_MIN_VALUE
580 if packagename not in currentPackageNames:
581 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
584 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
585 highestVersionCode = apk['versionCode']
586 if versionCode > highestVersionCode:
587 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
588 .format(integer=str(versionCode)))
590 obbsha256 = sha256sum(f)
591 obbs.append((packagename, versionCode, obbfile, obbsha256))
594 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
595 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
596 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
597 apk['obbMainFile'] = obbfile
598 apk['obbMainFileSha256'] = obbsha256
599 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
600 apk['obbPatchFile'] = obbfile
601 apk['obbPatchFileSha256'] = obbsha256
602 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
606 def translate_per_build_anti_features(apps, apks):
607 """Grab the anti-features list from the build metadata
609 For most Anti-Features, they are really most applicable per-APK,
610 not for an app. An app can fix a vulnerability, add/remove
611 tracking, etc. This reads the 'antifeatures' list from the Build
612 entries in the fdroiddata metadata file, then transforms it into
613 the 'antiFeatures' list of unique items for the index.
615 The field key is all lower case in the metadata file to match the
616 rest of the Build fields. It is 'antiFeatures' camel case in the
617 implementation, index, and fdroidclient since it is translated
618 from the build 'antifeatures' field, not directly included.
622 antiFeatures = dict()
623 for packageName, app in apps.items():
625 for build in app['builds']:
626 afl = build.get('antifeatures')
628 d[int(build.versionCode)] = afl
630 antiFeatures[packageName] = d
633 d = antiFeatures.get(apk['packageName'])
635 afl = d.get(apk['versionCode'])
637 apk['antiFeatures'].update(afl)
640 def _get_localized_dict(app, locale):
641 '''get the dict to add localized store metadata to'''
642 if 'localized' not in app:
643 app['localized'] = collections.OrderedDict()
644 if locale not in app['localized']:
645 app['localized'][locale] = collections.OrderedDict()
646 return app['localized'][locale]
649 def _set_localized_text_entry(app, locale, key, f):
650 limit = config['char_limits'][key]
651 localized = _get_localized_dict(app, locale)
653 text = fp.read()[:limit]
655 localized[key] = text
658 def _set_author_entry(app, key, f):
659 limit = config['char_limits']['author']
661 text = fp.read()[:limit]
666 def copy_triple_t_store_metadata(apps):
667 """Include store metadata from the app's source repo
669 The Triple-T Gradle Play Publisher is a plugin that has a standard
670 file layout for all of the metadata and graphics that the Google
671 Play Store accepts. Since F-Droid has the git repo, it can just
672 pluck those files directly. This method reads any text files into
673 the app dict, then copies any graphics into the fdroid repo
676 This needs to be run before insert_localized_app_metadata() so that
677 the graphics files that are copied into the fdroid repo get
680 https://github.com/Triple-T/gradle-play-publisher#upload-images
681 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
685 if not os.path.isdir('build'):
686 return # nothing to do
688 for packageName, app in apps.items():
689 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
690 logging.debug('Triple-T Gradle Play Publisher: ' + d)
691 for root, dirs, files in os.walk(d):
692 segments = root.split('/')
693 locale = segments[-2]
695 if f == 'fulldescription':
696 _set_localized_text_entry(app, locale, 'description',
697 os.path.join(root, f))
699 elif f == 'shortdescription':
700 _set_localized_text_entry(app, locale, 'summary',
701 os.path.join(root, f))
704 _set_localized_text_entry(app, locale, 'name',
705 os.path.join(root, f))
708 _set_localized_text_entry(app, locale, 'video',
709 os.path.join(root, f))
711 elif f == 'whatsnew':
712 _set_localized_text_entry(app, segments[-1], 'whatsNew',
713 os.path.join(root, f))
715 elif f == 'contactEmail':
716 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
718 elif f == 'contactPhone':
719 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
721 elif f == 'contactWebsite':
722 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
725 base, extension = common.get_extension(f)
726 dirname = os.path.basename(root)
727 if extension in ALLOWED_EXTENSIONS \
728 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
729 if segments[-2] == 'listing':
730 locale = segments[-3]
732 locale = segments[-2]
733 destdir = os.path.join('repo', packageName, locale, dirname)
734 os.makedirs(destdir, mode=0o755, exist_ok=True)
735 sourcefile = os.path.join(root, f)
736 destfile = os.path.join(destdir, os.path.basename(f))
737 logging.debug('copying ' + sourcefile + ' ' + destfile)
738 shutil.copy(sourcefile, destfile)
741 def insert_localized_app_metadata(apps):
742 """scans standard locations for graphics and localized text
744 Scans for localized description files, store graphics, and
745 screenshot PNG files in statically defined screenshots directory
746 and adds them to the app metadata. The screenshots and graphic
747 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
748 and must be in the following layout:
749 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
751 repo/packageName/locale/featureGraphic.png
752 repo/packageName/locale/phoneScreenshots/1.png
753 repo/packageName/locale/phoneScreenshots/2.png
755 The changelog files must be text files named with the versionCode
756 ending with ".txt" and must be in the following layout:
757 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
759 repo/packageName/locale/changelogs/12345.txt
761 This will scan the each app's source repo then the metadata/ dir
762 for these standard locations of changelog files. If it finds
763 them, they will be added to the dict of all packages, with the
764 versions in the metadata/ folder taking precendence over the what
765 is in the app's source repo.
767 Where "packageName" is the app's packageName and "locale" is the locale
768 of the graphics, e.g. what language they are in, using the IETF RFC5646
769 format (en-US, fr-CA, es-MX, etc).
771 This will also scan the app's git for a fastlane folder, and the
772 metadata/ folder and the apps' source repos for standard locations
773 of graphic and screenshot files. If it finds them, it will copy
774 them into the repo. The fastlane files follow this pattern:
775 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
779 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
780 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
781 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
782 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
784 for srcd in sorted(sourcedirs):
785 if not os.path.isdir(srcd):
787 for root, dirs, files in os.walk(srcd):
788 segments = root.split('/')
789 packageName = segments[1]
790 if packageName not in apps:
791 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
793 locale = segments[-1]
794 destdir = os.path.join('repo', packageName, locale)
796 # flavours specified in build receipt
798 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
799 and 'gradle' in apps[packageName].builds[-1]:
800 build_flavours = apps[packageName].builds[-1].gradle
802 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
803 logging.debug("ignoring due to wrong flavour")
807 if f in ('description.txt', 'full_description.txt'):
808 _set_localized_text_entry(apps[packageName], locale, 'description',
809 os.path.join(root, f))
811 elif f in ('summary.txt', 'short_description.txt'):
812 _set_localized_text_entry(apps[packageName], locale, 'summary',
813 os.path.join(root, f))
815 elif f in ('name.txt', 'title.txt'):
816 _set_localized_text_entry(apps[packageName], locale, 'name',
817 os.path.join(root, f))
819 elif f == 'video.txt':
820 _set_localized_text_entry(apps[packageName], locale, 'video',
821 os.path.join(root, f))
823 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
824 locale = segments[-2]
825 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
826 os.path.join(root, f))
829 base, extension = common.get_extension(f)
830 if locale == 'images':
831 locale = segments[-2]
832 destdir = os.path.join('repo', packageName, locale)
833 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
834 os.makedirs(destdir, mode=0o755, exist_ok=True)
835 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
836 shutil.copy(os.path.join(root, f), destdir)
838 if d in SCREENSHOT_DIRS:
839 if locale == 'images':
840 locale = segments[-2]
841 destdir = os.path.join('repo', packageName, locale)
842 for f in glob.glob(os.path.join(root, d, '*.*')):
843 _ignored, extension = common.get_extension(f)
844 if extension in ALLOWED_EXTENSIONS:
845 screenshotdestdir = os.path.join(destdir, d)
846 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
847 logging.debug('copying ' + f + ' ' + screenshotdestdir)
848 shutil.copy(f, screenshotdestdir)
850 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
852 if not os.path.isdir(d):
854 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
855 if not os.path.isfile(f):
857 segments = f.split('/')
858 packageName = segments[1]
860 screenshotdir = segments[3]
861 filename = os.path.basename(f)
862 base, extension = common.get_extension(filename)
864 if packageName not in apps:
865 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
866 .format(path=filename, name=packageName))
868 graphics = _get_localized_dict(apps[packageName], locale)
870 if extension not in ALLOWED_EXTENSIONS:
871 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
872 elif base in GRAPHIC_NAMES:
873 # there can only be zero or one of these per locale
874 graphics[base] = filename
875 elif screenshotdir in SCREENSHOT_DIRS:
876 # there can any number of these per locale
877 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
878 if screenshotdir not in graphics:
879 graphics[screenshotdir] = []
880 graphics[screenshotdir].append(filename)
882 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
885 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
886 """Scan a repo for all files with an extension except APK/OBB
888 :param apkcache: current cached info about all repo files
889 :param repodir: repo directory to scan
890 :param knownapks: list of all known files, as per metadata.read_metadata
891 :param use_date_from_file: use date from file (instead of current date)
892 for newly added files
897 repodir = repodir.encode('utf-8')
898 for name in os.listdir(repodir):
899 file_extension = common.get_file_extension(name)
900 if file_extension == 'apk' or file_extension == 'obb':
902 filename = os.path.join(repodir, name)
903 name_utf8 = name.decode('utf-8')
904 if filename.endswith(b'_src.tar.gz'):
905 logging.debug(_('skipping source tarball: {path}')
906 .format(path=filename.decode('utf-8')))
908 if not common.is_repo_file(filename):
910 stat = os.stat(filename)
911 if stat.st_size == 0:
912 raise FDroidException(_('{path} is zero size!')
913 .format(path=filename))
915 shasum = sha256sum(filename)
918 repo_file = apkcache[name]
919 # added time is cached as tuple but used here as datetime instance
920 if 'added' in repo_file:
921 a = repo_file['added']
922 if isinstance(a, datetime):
923 repo_file['added'] = a
925 repo_file['added'] = datetime(*a[:6])
926 if repo_file.get('hash') == shasum:
927 logging.debug(_("Reading {apkfilename} from cache")
928 .format(apkfilename=name_utf8))
931 logging.debug(_("Ignoring stale cache data for {apkfilename}")
932 .format(apkfilename=name_utf8))
935 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
936 repo_file = collections.OrderedDict()
937 repo_file['name'] = os.path.splitext(name_utf8)[0]
938 # TODO rename apkname globally to something more generic
939 repo_file['apkName'] = name_utf8
940 repo_file['hash'] = shasum
941 repo_file['hashType'] = 'sha256'
942 repo_file['versionCode'] = 0
943 repo_file['versionName'] = shasum
944 # the static ID is the SHA256 unless it is set in the metadata
945 repo_file['packageName'] = shasum
947 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
949 repo_file['packageName'] = m.group(1)
950 repo_file['versionCode'] = int(m.group(2))
951 srcfilename = name + b'_src.tar.gz'
952 if os.path.exists(os.path.join(repodir, srcfilename)):
953 repo_file['srcname'] = srcfilename.decode('utf-8')
954 repo_file['size'] = stat.st_size
956 apkcache[name] = repo_file
959 if use_date_from_file:
960 timestamp = stat.st_ctime
961 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
963 default_date_param = None
965 # Record in knownapks, getting the added date at the same time..
966 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
967 default_date=default_date_param)
969 repo_file['added'] = added
971 repo_files.append(repo_file)
973 return repo_files, cachechanged
976 def scan_apk(apk_file):
978 Scans an APK file and returns dictionary with metadata of the APK.
980 Attention: This does *not* verify that the APK signature is correct.
982 :param apk_file: The (ideally absolute) path to the APK file
983 :raises BuildException
984 :return A dict containing APK metadata
987 'hash': sha256sum(apk_file),
988 'hashType': 'sha256',
989 'uses-permission': [],
990 'uses-permission-sdk-23': [],
994 'antiFeatures': set(),
997 if SdkToolsPopen(['aapt', 'version'], output=False):
998 scan_apk_aapt(apk, apk_file)
1000 scan_apk_androguard(apk, apk_file)
1002 # Get the signature, or rather the signing key fingerprints
1003 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1004 apk['sig'] = getsig(apk_file)
1006 raise BuildException("Failed to get apk signature")
1007 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1009 if not apk.get('signer'):
1010 raise BuildException("Failed to get apk signing key fingerprint")
1012 # Get size of the APK
1013 apk['size'] = os.path.getsize(apk_file)
1015 if 'minSdkVersion' not in apk:
1016 logging.warning("No SDK version information found in {0}".format(apk_file))
1017 apk['minSdkVersion'] = 1
1019 # Check for known vulnerabilities
1020 if has_known_vulnerability(apk_file):
1021 apk['antiFeatures'].add('KnownVuln')
1026 def scan_apk_aapt(apk, apkfile):
1027 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1028 if p.returncode != 0:
1029 if options.delete_unknown:
1030 if os.path.exists(apkfile):
1031 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1034 logging.error("Could not find {0} to remove it".format(apkfile))
1036 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1037 raise BuildException(_("Invalid APK"))
1038 for line in p.output.splitlines():
1039 if line.startswith("package:"):
1041 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1042 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1043 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1044 except Exception as e:
1045 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1046 elif line.startswith("application:"):
1047 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1048 # Keep path to non-dpi icon in case we need it
1049 match = re.match(APK_ICON_PAT_NODPI, line)
1051 apk['icons_src']['-1'] = match.group(1)
1052 elif line.startswith("launchable-activity:"):
1053 # Only use launchable-activity as fallback to application
1055 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1056 if '-1' not in apk['icons_src']:
1057 match = re.match(APK_ICON_PAT_NODPI, line)
1059 apk['icons_src']['-1'] = match.group(1)
1060 elif line.startswith("application-icon-"):
1061 match = re.match(APK_ICON_PAT, line)
1063 density = match.group(1)
1064 path = match.group(2)
1065 apk['icons_src'][density] = path
1066 elif line.startswith("sdkVersion:"):
1067 m = re.match(APK_SDK_VERSION_PAT, line)
1069 logging.error(line.replace('sdkVersion:', '')
1070 + ' is not a valid minSdkVersion!')
1072 apk['minSdkVersion'] = m.group(1)
1073 # if target not set, default to min
1074 if 'targetSdkVersion' not in apk:
1075 apk['targetSdkVersion'] = m.group(1)
1076 elif line.startswith("targetSdkVersion:"):
1077 m = re.match(APK_SDK_VERSION_PAT, line)
1079 logging.error(line.replace('targetSdkVersion:', '')
1080 + ' is not a valid targetSdkVersion!')
1082 apk['targetSdkVersion'] = m.group(1)
1083 elif line.startswith("maxSdkVersion:"):
1084 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1085 elif line.startswith("native-code:"):
1086 apk['nativecode'] = []
1087 for arch in line[13:].split(' '):
1088 apk['nativecode'].append(arch[1:-1])
1089 elif line.startswith('uses-permission:'):
1090 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1091 if perm_match['maxSdkVersion']:
1092 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1093 permission = UsesPermission(
1095 perm_match['maxSdkVersion']
1098 apk['uses-permission'].append(permission)
1099 elif line.startswith('uses-permission-sdk-23:'):
1100 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1101 if perm_match['maxSdkVersion']:
1102 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1103 permission_sdk_23 = UsesPermissionSdk23(
1105 perm_match['maxSdkVersion']
1108 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1110 elif line.startswith('uses-feature:'):
1111 feature = re.match(APK_FEATURE_PAT, line).group(1)
1112 # Filter out this, it's only added with the latest SDK tools and
1113 # causes problems for lots of apps.
1114 if feature != "android.hardware.screen.portrait" \
1115 and feature != "android.hardware.screen.landscape":
1116 if feature.startswith("android.feature."):
1117 feature = feature[16:]
1118 apk['features'].add(feature)
1121 def scan_apk_androguard(apk, apkfile):
1123 from androguard.core.bytecodes.apk import APK
1124 apkobject = APK(apkfile)
1125 if apkobject.is_valid_APK():
1126 arsc = apkobject.get_android_resources()
1128 if options.delete_unknown:
1129 if os.path.exists(apkfile):
1130 logging.error(_("Failed to get apk information, deleting {path}")
1131 .format(path=apkfile))
1134 logging.error(_("Could not find {path} to remove it")
1135 .format(path=apkfile))
1137 logging.error(_("Failed to get apk information, skipping {path}")
1138 .format(path=apkfile))
1139 raise BuildException(_("Invalid APK"))
1141 raise FDroidException("androguard library is not installed and aapt not present")
1142 except FileNotFoundError:
1143 logging.error(_("Could not open apk file for analysis"))
1144 raise BuildException(_("Invalid APK"))
1146 apk['packageName'] = apkobject.get_package()
1147 apk['versionCode'] = int(apkobject.get_androidversion_code())
1148 apk['versionName'] = apkobject.get_androidversion_name()
1149 if apk['versionName'][0] == "@":
1150 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1151 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1152 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1153 apk['name'] = apkobject.get_app_name()
1155 if apkobject.get_max_sdk_version() is not None:
1156 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1157 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1158 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1160 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1161 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1163 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1165 for file in apkobject.get_files():
1166 d_re = density_re.match(file)
1168 folder = d_re.group(1).split('-')
1170 resolution = folder[1]
1173 density = screen_resolutions[resolution]
1174 apk['icons_src'][density] = d_re.group(0)
1176 if apk['icons_src'].get('-1') is None:
1177 apk['icons_src']['-1'] = apk['icons_src']['160']
1179 arch_re = re.compile("^lib/(.*)/.*$")
1180 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1182 apk['nativecode'] = []
1183 apk['nativecode'].extend(sorted(list(arch)))
1185 xml = apkobject.get_android_manifest_xml()
1187 for item in xml.getElementsByTagName('uses-permission'):
1188 name = str(item.getAttribute("android:name"))
1189 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1190 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1191 permission = UsesPermission(
1195 apk['uses-permission'].append(permission)
1197 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1198 name = str(item.getAttribute("android:name"))
1199 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1200 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1201 permission_sdk_23 = UsesPermissionSdk23(
1205 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1207 for item in xml.getElementsByTagName('uses-feature'):
1208 feature = str(item.getAttribute("android:name"))
1209 if feature != "android.hardware.screen.portrait" \
1210 and feature != "android.hardware.screen.landscape":
1211 if feature.startswith("android.feature."):
1212 feature = feature[16:]
1213 apk['features'].append(feature)
1216 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1217 allow_disabled_algorithms=False, archive_bad_sig=False):
1218 """Processes the apk with the given filename in the given repo directory.
1220 This also extracts the icons.
1222 :param apkcache: current apk cache information
1223 :param apkfilename: the filename of the apk to scan
1224 :param repodir: repo directory to scan
1225 :param knownapks: known apks info
1226 :param use_date_from_apk: use date from APK (instead of current date)
1227 for newly added APKs
1228 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1229 disabled algorithms in the signature (e.g. MD5)
1230 :param archive_bad_sig: move APKs with a bad signature to the archive
1231 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1232 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1236 apkfile = os.path.join(repodir, apkfilename)
1238 cachechanged = False
1240 if apkfilename in apkcache:
1241 apk = apkcache[apkfilename]
1242 if apk.get('hash') == sha256sum(apkfile):
1243 logging.debug(_("Reading {apkfilename} from cache")
1244 .format(apkfilename=apkfilename))
1247 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1248 .format(apkfilename=apkfilename))
1251 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1254 apk = scan_apk(apkfile)
1255 except BuildException:
1256 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1257 .format(apkfilename=apkfilename))
1258 return True, None, False
1260 # Check for debuggable apks...
1261 if common.isApkAndDebuggable(apkfile):
1262 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1264 if options.rename_apks:
1265 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1266 std_short_name = os.path.join(repodir, n)
1267 if apkfile != std_short_name:
1268 if os.path.exists(std_short_name):
1269 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1270 if apkfile != std_long_name:
1271 if os.path.exists(std_long_name):
1272 dupdir = os.path.join('duplicates', repodir)
1273 if not os.path.isdir(dupdir):
1274 os.makedirs(dupdir, exist_ok=True)
1275 dupfile = os.path.join('duplicates', std_long_name)
1276 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1277 os.rename(apkfile, dupfile)
1278 return True, None, False
1280 os.rename(apkfile, std_long_name)
1281 apkfile = std_long_name
1283 os.rename(apkfile, std_short_name)
1284 apkfile = std_short_name
1285 apkfilename = apkfile[len(repodir) + 1:]
1287 apk['apkName'] = apkfilename
1288 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1289 if os.path.exists(os.path.join(repodir, srcfilename)):
1290 apk['srcname'] = srcfilename
1292 # verify the jar signature is correct, allow deprecated
1293 # algorithms only if the APK is in the archive.
1295 if not common.verify_apk_signature(apkfile):
1296 if repodir == 'archive' or allow_disabled_algorithms:
1297 if common.verify_old_apk_signature(apkfile):
1298 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1306 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1307 .format(apkfilename=apkfilename))
1308 move_apk_between_sections(repodir, 'archive', apk)
1310 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1311 .format(apkfilename=apkfilename))
1312 return True, None, False
1314 apkzip = zipfile.ZipFile(apkfile, 'r')
1316 manifest = apkzip.getinfo('AndroidManifest.xml')
1317 if manifest.date_time[1] == 0: # month can't be zero
1318 logging.debug(_('AndroidManifest.xml has no date'))
1320 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1322 # extract icons from APK zip file
1323 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1325 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1327 apkzip.close() # ensure that APK zip file gets closed
1329 # resize existing icons for densities missing in the APK
1330 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1332 if use_date_from_apk and manifest.date_time[1] != 0:
1333 default_date_param = datetime(*manifest.date_time)
1335 default_date_param = None
1337 # Record in known apks, getting the added date at the same time..
1338 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1339 default_date=default_date_param)
1341 apk['added'] = added
1343 apkcache[apkfilename] = apk
1346 return False, apk, cachechanged
1349 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1350 """Processes the apks in the given repo directory.
1352 This also extracts the icons.
1354 :param apkcache: current apk cache information
1355 :param repodir: repo directory to scan
1356 :param knownapks: known apks info
1357 :param use_date_from_apk: use date from APK (instead of current date)
1358 for newly added APKs
1359 :returns: (apks, cachechanged) where apks is a list of apk information,
1360 and cachechanged is True if the apkcache got changed.
1363 cachechanged = False
1365 for icon_dir in get_all_icon_dirs(repodir):
1366 if os.path.exists(icon_dir):
1368 shutil.rmtree(icon_dir)
1369 os.makedirs(icon_dir)
1371 os.makedirs(icon_dir)
1374 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1375 apkfilename = apkfile[len(repodir) + 1:]
1376 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1377 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1378 use_date_from_apk, ada, True)
1382 cachechanged = cachechanged or cachethis
1384 return apks, cachechanged
1387 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1389 Extracts icons from the given APK zip in various densities,
1390 saves them into given repo directory
1391 and stores their names in the APK metadata dictionary.
1393 :param icon_filename: A string representing the icon's file name
1394 :param apk: A populated dictionary containing APK metadata.
1395 Needs to have 'icons_src' key
1396 :param apkzip: An opened zipfile.ZipFile of the APK file
1397 :param repo_dir: The directory of the APK's repository
1398 :return: A list of icon densities that are missing
1400 empty_densities = []
1401 for density in screen_densities:
1402 if density not in apk['icons_src']:
1403 empty_densities.append(density)
1405 icon_src = apk['icons_src'][density]
1406 icon_dir = get_icon_dir(repo_dir, density)
1407 icon_dest = os.path.join(icon_dir, icon_filename)
1409 # Extract the icon files per density
1410 if icon_src.endswith('.xml'):
1411 png = os.path.basename(icon_src)[:-4] + '.png'
1412 for f in apkzip.namelist():
1414 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1415 if m and screen_resolutions[m.group(2)] == density:
1417 if icon_src.endswith('.xml'):
1418 empty_densities.append(density)
1421 with open(icon_dest, 'wb') as f:
1422 f.write(get_icon_bytes(apkzip, icon_src))
1423 apk['icons'][density] = icon_filename
1424 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1425 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1426 del apk['icons_src'][density]
1427 empty_densities.append(density)
1429 if '-1' in apk['icons_src']:
1430 icon_src = apk['icons_src']['-1']
1431 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1432 with open(icon_path, 'wb') as f:
1433 f.write(get_icon_bytes(apkzip, icon_src))
1435 im = Image.open(icon_path)
1436 dpi = px_to_dpi(im.size[0])
1437 for density in screen_densities:
1438 if density in apk['icons']:
1440 if density == screen_densities[-1] or dpi >= int(density):
1441 apk['icons'][density] = icon_filename
1442 shutil.move(icon_path,
1443 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1444 empty_densities.remove(density)
1446 except Exception as e:
1447 logging.warning(_("Failed reading {path}: {error}")
1448 .format(path=icon_path, error=e))
1451 apk['icon'] = icon_filename
1453 return empty_densities
1456 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1458 Resize existing icons for densities missing in the APK to ensure all densities are available
1460 :param empty_densities: A list of icon densities that are missing
1461 :param icon_filename: A string representing the icon's file name
1462 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1463 :param repo_dir: The directory of the APK's repository
1465 # First try resizing down to not lose quality
1467 for density in screen_densities:
1468 if density not in empty_densities:
1469 last_density = density
1471 if last_density is None:
1473 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1475 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1476 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1479 fp = open(last_icon_path, 'rb')
1482 size = dpi_to_px(density)
1484 im.thumbnail((size, size), Image.ANTIALIAS)
1485 im.save(icon_path, "PNG")
1486 empty_densities.remove(density)
1487 except Exception as e:
1488 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1493 # Then just copy from the highest resolution available
1495 for density in reversed(screen_densities):
1496 if density not in empty_densities:
1497 last_density = density
1500 if last_density is None:
1504 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1505 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1507 empty_densities.remove(density)
1509 for density in screen_densities:
1510 icon_dir = get_icon_dir(repo_dir, density)
1511 icon_dest = os.path.join(icon_dir, icon_filename)
1512 resize_icon(icon_dest, density)
1514 # Copy from icons-mdpi to icons since mdpi is the baseline density
1515 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1516 if os.path.isfile(baseline):
1517 apk['icons']['0'] = icon_filename
1518 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1521 def apply_info_from_latest_apk(apps, apks):
1523 Some information from the apks needs to be applied up to the application level.
1524 When doing this, we use the info from the most recent version's apk.
1525 We deal with figuring out when the app was added and last updated at the same time.
1527 for appid, app in apps.items():
1528 bestver = UNSET_VERSION_CODE
1530 if apk['packageName'] == appid:
1531 if apk['versionCode'] > bestver:
1532 bestver = apk['versionCode']
1536 if not app.added or apk['added'] < app.added:
1537 app.added = apk['added']
1538 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1539 app.lastUpdated = apk['added']
1542 logging.debug("Don't know when " + appid + " was added")
1543 if not app.lastUpdated:
1544 logging.debug("Don't know when " + appid + " was last updated")
1546 if bestver == UNSET_VERSION_CODE:
1548 if app.Name is None:
1549 app.Name = app.AutoName or appid
1551 logging.debug("Application " + appid + " has no packages")
1553 if app.Name is None:
1554 app.Name = bestapk['name']
1555 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1556 if app.CurrentVersionCode is None:
1557 app.CurrentVersionCode = str(bestver)
1560 def make_categories_txt(repodir, categories):
1561 '''Write a category list in the repo to allow quick access'''
1563 for cat in sorted(categories):
1564 catdata += cat + '\n'
1565 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1569 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1571 def filter_apk_list_sorted(apk_list):
1573 for apk in apk_list:
1574 if apk['packageName'] == appid:
1577 # Sort the apk list by version code. First is highest/newest.
1578 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1580 for appid, app in apps.items():
1582 if app.ArchivePolicy:
1583 keepversions = int(app.ArchivePolicy[:-9])
1585 keepversions = defaultkeepversions
1587 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1588 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1590 current_app_apks = filter_apk_list_sorted(apks)
1591 if len(current_app_apks) > keepversions:
1592 # Move back the ones we don't want.
1593 for apk in current_app_apks[keepversions:]:
1594 move_apk_between_sections(repodir, archivedir, apk)
1595 archapks.append(apk)
1598 current_app_archapks = filter_apk_list_sorted(archapks)
1599 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1601 # Move forward the ones we want again, except DisableAlgorithm
1602 for apk in current_app_archapks:
1603 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1604 move_apk_between_sections(archivedir, repodir, apk)
1605 archapks.remove(apk)
1608 if kept == keepversions:
1612 def move_apk_between_sections(from_dir, to_dir, apk):
1613 """move an APK from repo to archive or vice versa"""
1615 def _move_file(from_dir, to_dir, filename, ignore_missing):
1616 from_path = os.path.join(from_dir, filename)
1617 if ignore_missing and not os.path.exists(from_path):
1619 to_path = os.path.join(to_dir, filename)
1620 if not os.path.exists(to_dir):
1622 shutil.move(from_path, to_path)
1624 if from_dir == to_dir:
1627 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1628 _move_file(from_dir, to_dir, apk['apkName'], False)
1629 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1630 for density in all_screen_densities:
1631 from_icon_dir = get_icon_dir(from_dir, density)
1632 to_icon_dir = get_icon_dir(to_dir, density)
1633 if density not in apk.get('icons', []):
1635 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1636 if 'srcname' in apk:
1637 _move_file(from_dir, to_dir, apk['srcname'], False)
1640 def add_apks_to_per_app_repos(repodir, apks):
1641 apks_per_app = dict()
1643 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1644 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1645 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1646 apks_per_app[apk['packageName']] = apk
1648 if not os.path.exists(apk['per_app_icons']):
1649 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1650 os.makedirs(apk['per_app_icons'])
1652 apkpath = os.path.join(repodir, apk['apkName'])
1653 shutil.copy(apkpath, apk['per_app_repo'])
1654 apksigpath = apkpath + '.sig'
1655 if os.path.exists(apksigpath):
1656 shutil.copy(apksigpath, apk['per_app_repo'])
1657 apkascpath = apkpath + '.asc'
1658 if os.path.exists(apkascpath):
1659 shutil.copy(apkascpath, apk['per_app_repo'])
1662 def create_metadata_from_template(apk):
1663 '''create a new metadata file using internal or external template
1665 Generate warnings for apk's with no metadata (or create skeleton
1666 metadata files, if requested on the command line). Though the
1667 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1668 since those impose things on the metadata file made from the
1669 template: field sort order, empty field value, formatting, etc.
1673 if os.path.exists('template.yml'):
1674 with open('template.yml') as f:
1676 if 'name' in apk and apk['name'] != '':
1677 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1678 r'\1 ' + apk['name'],
1680 flags=re.IGNORECASE | re.MULTILINE)
1682 logging.warning(_('{appid} does not have a name! Using package name instead.')
1683 .format(appid=apk['packageName']))
1684 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1685 r'\1 ' + apk['packageName'],
1687 flags=re.IGNORECASE | re.MULTILINE)
1688 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1692 app['Categories'] = [os.path.basename(os.getcwd())]
1693 # include some blanks as part of the template
1694 app['AuthorName'] = ''
1697 app['IssueTracker'] = ''
1698 app['SourceCode'] = ''
1699 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1700 if 'name' in apk and apk['name'] != '':
1701 app['Name'] = apk['name']
1703 logging.warning(_('{appid} does not have a name! Using package name instead.')
1704 .format(appid=apk['packageName']))
1705 app['Name'] = apk['packageName']
1706 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1707 yaml.dump(app, f, default_flow_style=False)
1708 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1717 global config, options
1719 # Parse command line...
1720 parser = ArgumentParser()
1721 common.setup_global_opts(parser)
1722 parser.add_argument("--create-key", action="store_true", default=False,
1723 help=_("Add a repo signing key to an unsigned repo"))
1724 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1725 help=_("Add skeleton metadata files for APKs that are missing them"))
1726 parser.add_argument("--delete-unknown", action="store_true", default=False,
1727 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1728 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1729 help=_("Report on build data status"))
1730 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1731 help=_("Interactively ask about things that need updating."))
1732 parser.add_argument("-I", "--icons", action="store_true", default=False,
1733 help=_("Resize all the icons exceeding the max pixel size and exit"))
1734 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1735 help=_("Specify editor to use in interactive mode. Default " +
1736 "is {path}").format(path='/etc/alternatives/editor'))
1737 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1738 help=_("Update the wiki"))
1739 parser.add_argument("--pretty", action="store_true", default=False,
1740 help=_("Produce human-readable XML/JSON for index files"))
1741 parser.add_argument("--clean", action="store_true", default=False,
1742 help=_("Clean update - don't uses caches, reprocess all APKs"))
1743 parser.add_argument("--nosign", action="store_true", default=False,
1744 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1745 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1746 help=_("Use date from APK instead of current time for newly added APKs"))
1747 parser.add_argument("--rename-apks", action="store_true", default=False,
1748 help=_("Rename APK files that do not match package.name_123.apk"))
1749 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1750 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1751 metadata.add_metadata_arguments(parser)
1752 options = parser.parse_args()
1753 metadata.warnings_action = options.W
1755 config = common.read_config(options)
1757 if not ('jarsigner' in config and 'keytool' in config):
1758 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1761 if config['archive_older'] != 0:
1762 repodirs.append('archive')
1763 if not os.path.exists('archive'):
1767 resize_all_icons(repodirs)
1770 if options.rename_apks:
1771 options.clean = True
1773 # check that icons exist now, rather than fail at the end of `fdroid update`
1774 for k in ['repo_icon', 'archive_icon']:
1776 if not os.path.exists(config[k]):
1777 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1778 .format(name=k, path=config[k]))
1781 # if the user asks to create a keystore, do it now, reusing whatever it can
1782 if options.create_key:
1783 if os.path.exists(config['keystore']):
1784 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1785 logging.critical("\t'" + config['keystore'] + "'")
1788 if 'repo_keyalias' not in config:
1789 config['repo_keyalias'] = socket.getfqdn()
1790 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1791 if 'keydname' not in config:
1792 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1793 common.write_to_config(config, 'keydname', config['keydname'])
1794 if 'keystore' not in config:
1795 config['keystore'] = common.default_config['keystore']
1796 common.write_to_config(config, 'keystore', config['keystore'])
1798 password = common.genpassword()
1799 if 'keystorepass' not in config:
1800 config['keystorepass'] = password
1801 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1802 if 'keypass' not in config:
1803 config['keypass'] = password
1804 common.write_to_config(config, 'keypass', config['keypass'])
1805 common.genkeystore(config)
1808 apps = metadata.read_metadata()
1810 # Generate a list of categories...
1812 for app in apps.values():
1813 categories.update(app.Categories)
1815 # Read known apks data (will be updated and written back when we've finished)
1816 knownapks = common.KnownApks()
1819 apkcache = get_cache()
1821 # Delete builds for disabled apps
1822 delete_disabled_builds(apps, apkcache, repodirs)
1824 # Scan all apks in the main repo
1825 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1827 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1828 options.use_date_from_apk)
1829 cachechanged = cachechanged or fcachechanged
1832 if apk['packageName'] not in apps:
1833 if options.create_metadata:
1834 create_metadata_from_template(apk)
1835 apps = metadata.read_metadata()
1837 msg = _("{apkfilename} ({appid}) has no metadata!") \
1838 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1839 if options.delete_unknown:
1840 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1841 .format(apkfilename=apk['apkName']))
1842 rmf = os.path.join(repodirs[0], apk['apkName'])
1843 if not os.path.exists(rmf):
1844 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1848 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1850 copy_triple_t_store_metadata(apps)
1851 insert_obbs(repodirs[0], apps, apks)
1852 insert_localized_app_metadata(apps)
1853 translate_per_build_anti_features(apps, apks)
1855 # Scan the archive repo for apks as well
1856 if len(repodirs) > 1:
1857 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1863 # Apply information from latest apks to the application and update dates
1864 apply_info_from_latest_apk(apps, apks + archapks)
1866 # Sort the app list by name, then the web site doesn't have to by default.
1867 # (we had to wait until we'd scanned the apks to do this, because mostly the
1868 # name comes from there!)
1869 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1871 # APKs are placed into multiple repos based on the app package, providing
1872 # per-app subscription feeds for nightly builds and things like it
1873 if config['per_app_repos']:
1874 add_apks_to_per_app_repos(repodirs[0], apks)
1875 for appid, app in apps.items():
1876 repodir = os.path.join(appid, 'fdroid', 'repo')
1878 appdict[appid] = app
1879 if os.path.isdir(repodir):
1880 index.make(appdict, [appid], apks, repodir, False)
1882 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1885 if len(repodirs) > 1:
1886 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1888 # Make the index for the main repo...
1889 index.make(apps, sortedids, apks, repodirs[0], False)
1890 make_categories_txt(repodirs[0], categories)
1892 # If there's an archive repo, make the index for it. We already scanned it
1894 if len(repodirs) > 1:
1895 index.make(apps, sortedids, archapks, repodirs[1], True)
1897 git_remote = config.get('binary_transparency_remote')
1898 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1900 btlog.make_binary_transparency_log(repodirs)
1902 if config['update_stats']:
1903 # Update known apks info...
1904 knownapks.writeifchanged()
1906 # Generate latest apps data for widget
1907 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1909 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1911 appid = line.rstrip()
1912 data += appid + "\t"
1914 data += app.Name + "\t"
1915 if app.icon is not None:
1916 data += app.icon + "\t"
1917 data += app.License + "\n"
1918 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1922 write_cache(apkcache)
1924 # Update the wiki...
1926 update_wiki(apps, sortedids, apks + archapks)
1928 logging.info(_("Finished"))
1931 if __name__ == "__main__":