3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
98 return os.path.join(repodir, "icons")
99 return os.path.join(repodir, "icons-%s" % density)
102 def get_icon_dirs(repodir):
103 for density in screen_densities:
104 yield get_icon_dir(repodir, density)
107 def get_all_icon_dirs(repodir):
108 for density in all_screen_densities:
109 yield get_icon_dir(repodir, density)
112 def update_wiki(apps, sortedids, apks):
115 :param apps: fully populated list of all applications
116 :param apks: all apks, except...
118 logging.info("Updating wiki")
120 wikiredircat = 'App Redirects'
122 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
123 path=config['wiki_path'])
124 site.login(config['wiki_user'], config['wiki_password'])
126 generated_redirects = {}
128 for appid in sortedids:
129 app = metadata.App(apps[appid])
133 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
135 for af in sorted(app.AntiFeatures):
136 wikidata += '{{AntiFeature|' + af + '}}\n'
141 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
144 app.added.strftime('%Y-%m-%d') if app.added else '',
145 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
161 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
163 wikidata += app.Summary
164 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
166 wikidata += "=Description=\n"
167 wikidata += metadata.description_wiki(app.Description) + "\n"
169 wikidata += "=Maintainer Notes=\n"
170 if app.MaintainerNotes:
171 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
172 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
174 # Get a list of all packages for this application...
176 gotcurrentver = False
180 if apk['packageName'] == appid:
181 if str(apk['versionCode']) == app.CurrentVersionCode:
184 # Include ones we can't build, as a special case...
185 for build in app.builds:
187 if build.versionCode == app.CurrentVersionCode:
189 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
190 apklist.append({'versionCode': int(build.versionCode),
191 'versionName': build.versionName,
192 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
197 if apk['versionCode'] == int(build.versionCode):
202 apklist.append({'versionCode': int(build.versionCode),
203 'versionName': build.versionName,
204 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
206 if app.CurrentVersionCode == '0':
208 # Sort with most recent first...
209 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
211 wikidata += "=Versions=\n"
212 if len(apklist) == 0:
213 wikidata += "We currently have no versions of this app available."
214 elif not gotcurrentver:
215 wikidata += "We don't have the current version of this app."
217 wikidata += "We have the current version of this app."
218 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
219 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
220 if len(app.NoSourceSince) > 0:
221 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
222 if len(app.CurrentVersion) > 0:
223 wikidata += "The current (recommended) version is " + app.CurrentVersion
224 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
227 wikidata += "==" + apk['versionName'] + "==\n"
229 if 'buildproblem' in apk:
230 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
233 wikidata += "This version is built and signed by "
235 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
237 wikidata += "the original developer.\n\n"
238 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
240 wikidata += '\n[[Category:' + wikicat + ']]\n'
241 if len(app.NoSourceSince) > 0:
242 wikidata += '\n[[Category:Apps missing source code]]\n'
243 if validapks == 0 and not app.Disabled:
244 wikidata += '\n[[Category:Apps with no packages]]\n'
245 if cantupdate and not app.Disabled:
246 wikidata += "\n[[Category:Apps we cannot update]]\n"
247 if buildfails and not app.Disabled:
248 wikidata += "\n[[Category:Apps with failing builds]]\n"
249 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
250 wikidata += '\n[[Category:Apps to Update]]\n'
252 wikidata += '\n[[Category:Apps that are disabled]]\n'
253 if app.UpdateCheckMode == 'None' and not app.Disabled:
254 wikidata += '\n[[Category:Apps with no update check]]\n'
255 for appcat in app.Categories:
256 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
258 # We can't have underscores in the page name, even if they're in
259 # the package ID, because MediaWiki messes with them...
260 pagename = appid.replace('_', ' ')
262 # Drop a trailing newline, because mediawiki is going to drop it anyway
263 # and it we don't we'll think the page has changed when it hasn't...
264 if wikidata.endswith('\n'):
265 wikidata = wikidata[:-1]
267 generated_pages[pagename] = wikidata
269 # Make a redirect from the name to the ID too, unless there's
270 # already an existing page with the name and it isn't a redirect.
272 apppagename = app.Name.replace('_', ' ')
273 apppagename = apppagename.replace('{', '')
274 apppagename = apppagename.replace('}', ' ')
275 apppagename = apppagename.replace(':', ' ')
276 apppagename = apppagename.replace('[', ' ')
277 apppagename = apppagename.replace(']', ' ')
278 # Drop double spaces caused mostly by replacing ':' above
279 apppagename = apppagename.replace(' ', ' ')
280 for expagename in site.allpages(prefix=apppagename,
281 filterredir='nonredirects',
283 if expagename == apppagename:
285 # Another reason not to make the redirect page is if the app name
286 # is the same as it's ID, because that will overwrite the real page
287 # with an redirect to itself! (Although it seems like an odd
288 # scenario this happens a lot, e.g. where there is metadata but no
289 # builds or binaries to extract a name from.
290 if apppagename == pagename:
293 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
295 for tcat, genp in [(wikicat, generated_pages),
296 (wikiredircat, generated_redirects)]:
297 catpages = site.Pages['Category:' + tcat]
299 for page in catpages:
300 existingpages.append(page.name)
301 if page.name in genp:
302 pagetxt = page.edit()
303 if pagetxt != genp[page.name]:
304 logging.debug("Updating modified page " + page.name)
305 page.save(genp[page.name], summary='Auto-updated')
307 logging.debug("Page " + page.name + " is unchanged")
309 logging.warn("Deleting page " + page.name)
310 page.delete('No longer published')
311 for pagename, text in genp.items():
312 logging.debug("Checking " + pagename)
313 if pagename not in existingpages:
314 logging.debug("Creating page " + pagename)
316 newpage = site.Pages[pagename]
317 newpage.save(text, summary='Auto-created')
318 except Exception as e:
319 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
321 # Purge server cache to ensure counts are up to date
322 site.pages['Repository Maintenance'].purge()
325 def delete_disabled_builds(apps, apkcache, repodirs):
326 """Delete disabled build outputs.
328 :param apps: list of all applications, as per metadata.read_metadata
329 :param apkcache: current apk cache information
330 :param repodirs: the repo directories to process
332 for appid, app in apps.items():
333 for build in app['builds']:
334 if not build.disable:
336 apkfilename = common.get_release_filename(app, build)
337 iconfilename = "%s.%s.png" % (
340 for repodir in repodirs:
342 os.path.join(repodir, apkfilename),
343 os.path.join(repodir, apkfilename + '.asc'),
344 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
346 for density in all_screen_densities:
347 repo_dir = get_icon_dir(repodir, density)
348 files.append(os.path.join(repo_dir, iconfilename))
351 if os.path.exists(f):
352 logging.info("Deleting disabled build output " + f)
354 if apkfilename in apkcache:
355 del apkcache[apkfilename]
358 def resize_icon(iconpath, density):
360 if not os.path.isfile(iconpath):
365 fp = open(iconpath, 'rb')
367 size = dpi_to_px(density)
369 if any(length > size for length in im.size):
371 im.thumbnail((size, size), Image.ANTIALIAS)
372 logging.debug("%s was too large at %s - new size is %s" % (
373 iconpath, oldsize, im.size))
374 im.save(iconpath, "PNG")
376 except Exception as e:
377 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
384 def resize_all_icons(repodirs):
385 """Resize all icons that exceed the max size
387 :param repodirs: the repo directories to process
389 for repodir in repodirs:
390 for density in screen_densities:
391 icon_dir = get_icon_dir(repodir, density)
392 icon_glob = os.path.join(icon_dir, '*.png')
393 for iconpath in glob.glob(icon_glob):
394 resize_icon(iconpath, density)
398 """ Get the signing certificate of an apk. To get the same md5 has that
399 Android gets, we encode the .RSA certificate in a specific format and pass
400 it hex-encoded to the md5 digest algorithm.
402 :param apkpath: path to the apk
403 :returns: A string containing the md5 of the signature of the apk or None
404 if an error occurred.
407 with zipfile.ZipFile(apkpath, 'r') as apk:
408 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
411 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
414 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
417 cert = apk.read(certs[0])
419 cert_encoded = common.get_certificate(cert)
421 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
424 def get_cache_file():
425 return os.path.join('tmp', 'apkcache')
429 """Get the cached dict of the APK index
431 Gather information about all the apk files in the repo directory,
432 using cached data if possible. Some of the index operations take a
433 long time, like calculating the SHA-256 and verifying the APK
436 The cache is invalidated if the metadata version is different, or
437 the 'allow_disabled_algorithms' config/option is different. In
438 those cases, there is no easy way to know what has changed from
439 the cache, so just rerun the whole thing.
444 apkcachefile = get_cache_file()
445 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
446 if not options.clean and os.path.exists(apkcachefile):
447 with open(apkcachefile, 'rb') as cf:
448 apkcache = pickle.load(cf, encoding='utf-8')
449 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
450 or apkcache.get('allow_disabled_algorithms') != ada:
455 apkcache["METADATA_VERSION"] = METADATA_VERSION
456 apkcache['allow_disabled_algorithms'] = ada
461 def write_cache(apkcache):
462 apkcachefile = get_cache_file()
463 cache_path = os.path.dirname(apkcachefile)
464 if not os.path.exists(cache_path):
465 os.makedirs(cache_path)
466 with open(apkcachefile, 'wb') as cf:
467 pickle.dump(apkcache, cf)
470 def get_icon_bytes(apkzip, iconsrc):
471 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
473 return apkzip.read(iconsrc)
475 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
478 def sha256sum(filename):
479 '''Calculate the sha256 of the given file'''
480 sha = hashlib.sha256()
481 with open(filename, 'rb') as f:
487 return sha.hexdigest()
490 def has_known_vulnerability(filename):
491 """checks for known vulnerabilities in the APK
493 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
494 version. Google also enforces this:
495 https://support.google.com/faqs/answer/6376725?hl=en
497 Checks whether there are more than one classes.dex or AndroidManifest.xml
498 files, which is invalid and an essential part of the "Master Key" attack.
500 http://www.saurik.com/id/17
503 # statically load this pattern
504 if not hasattr(has_known_vulnerability, "pattern"):
505 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
508 with zipfile.ZipFile(filename) as zf:
509 for name in zf.namelist():
510 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
513 chunk = lib.read(4096)
516 m = has_known_vulnerability.pattern.search(chunk)
518 version = m.group(1).decode('ascii')
519 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
520 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
521 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
522 logging.debug(_('"{path}" contains recent {name} ({version})')
523 .format(path=filename, name=name, version=version))
525 logging.warning(_('"{path}" contains outdated {name} ({version})')
526 .format(path=filename, name=name, version=version))
529 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
530 if name in files_in_apk:
532 files_in_apk.add(name)
537 def insert_obbs(repodir, apps, apks):
538 """Scans the .obb files in a given repo directory and adds them to the
539 relevant APK instances. OBB files have versionCodes like APK
540 files, and they are loosely associated. If there is an OBB file
541 present, then any APK with the same or higher versionCode will use
542 that OBB file. There are two OBB types: main and patch, each APK
543 can only have only have one of each.
545 https://developer.android.com/google/play/expansion-files.html
547 :param repodir: repo directory to scan
548 :param apps: list of current, valid apps
549 :param apks: current information on all APKs
553 def obbWarnDelete(f, msg):
554 logging.warning(msg + ' ' + f)
555 if options.delete_unknown:
556 logging.error(_("Deleting unknown file: {path}").format(path=f))
560 java_Integer_MIN_VALUE = -pow(2, 31)
561 currentPackageNames = apps.keys()
562 for f in glob.glob(os.path.join(repodir, '*.obb')):
563 obbfile = os.path.basename(f)
564 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
565 chunks = obbfile.split('.')
566 if chunks[0] != 'main' and chunks[0] != 'patch':
567 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
569 if not re.match(r'^-?[0-9]+$', chunks[1]):
570 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
571 .format(name=chunks[0]))
573 versionCode = int(chunks[1])
574 packagename = ".".join(chunks[2:-1])
576 highestVersionCode = java_Integer_MIN_VALUE
577 if packagename not in currentPackageNames:
578 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
581 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
582 highestVersionCode = apk['versionCode']
583 if versionCode > highestVersionCode:
584 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
585 .format(integer=str(versionCode)))
587 obbsha256 = sha256sum(f)
588 obbs.append((packagename, versionCode, obbfile, obbsha256))
591 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
592 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
593 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
594 apk['obbMainFile'] = obbfile
595 apk['obbMainFileSha256'] = obbsha256
596 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
597 apk['obbPatchFile'] = obbfile
598 apk['obbPatchFileSha256'] = obbsha256
599 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
603 def translate_per_build_anti_features(apps, apks):
604 """Grab the anti-features list from the build metadata
606 For most Anti-Features, they are really most applicable per-APK,
607 not for an app. An app can fix a vulnerability, add/remove
608 tracking, etc. This reads the 'antifeatures' list from the Build
609 entries in the fdroiddata metadata file, then transforms it into
610 the 'antiFeatures' list of unique items for the index.
612 The field key is all lower case in the metadata file to match the
613 rest of the Build fields. It is 'antiFeatures' camel case in the
614 implementation, index, and fdroidclient since it is translated
615 from the build 'antifeatures' field, not directly included.
619 antiFeatures = dict()
620 for packageName, app in apps.items():
622 for build in app['builds']:
623 afl = build.get('antifeatures')
625 d[int(build.versionCode)] = afl
627 antiFeatures[packageName] = d
630 d = antiFeatures.get(apk['packageName'])
632 afl = d.get(apk['versionCode'])
634 apk['antiFeatures'].update(afl)
637 def _get_localized_dict(app, locale):
638 '''get the dict to add localized store metadata to'''
639 if 'localized' not in app:
640 app['localized'] = collections.OrderedDict()
641 if locale not in app['localized']:
642 app['localized'][locale] = collections.OrderedDict()
643 return app['localized'][locale]
646 def _set_localized_text_entry(app, locale, key, f):
647 limit = config['char_limits'][key]
648 localized = _get_localized_dict(app, locale)
650 text = fp.read()[:limit]
652 localized[key] = text
655 def _set_author_entry(app, key, f):
656 limit = config['char_limits']['author']
658 text = fp.read()[:limit]
663 def copy_triple_t_store_metadata(apps):
664 """Include store metadata from the app's source repo
666 The Triple-T Gradle Play Publisher is a plugin that has a standard
667 file layout for all of the metadata and graphics that the Google
668 Play Store accepts. Since F-Droid has the git repo, it can just
669 pluck those files directly. This method reads any text files into
670 the app dict, then copies any graphics into the fdroid repo
673 This needs to be run before insert_localized_app_metadata() so that
674 the graphics files that are copied into the fdroid repo get
677 https://github.com/Triple-T/gradle-play-publisher#upload-images
678 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
682 if not os.path.isdir('build'):
683 return # nothing to do
685 for packageName, app in apps.items():
686 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
687 logging.debug('Triple-T Gradle Play Publisher: ' + d)
688 for root, dirs, files in os.walk(d):
689 segments = root.split('/')
690 locale = segments[-2]
692 if f == 'fulldescription':
693 _set_localized_text_entry(app, locale, 'description',
694 os.path.join(root, f))
696 elif f == 'shortdescription':
697 _set_localized_text_entry(app, locale, 'summary',
698 os.path.join(root, f))
701 _set_localized_text_entry(app, locale, 'name',
702 os.path.join(root, f))
705 _set_localized_text_entry(app, locale, 'video',
706 os.path.join(root, f))
708 elif f == 'whatsnew':
709 _set_localized_text_entry(app, segments[-1], 'whatsNew',
710 os.path.join(root, f))
712 elif f == 'contactEmail':
713 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
715 elif f == 'contactPhone':
716 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
718 elif f == 'contactWebsite':
719 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
722 base, extension = common.get_extension(f)
723 dirname = os.path.basename(root)
724 if extension in ALLOWED_EXTENSIONS \
725 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
726 if segments[-2] == 'listing':
727 locale = segments[-3]
729 locale = segments[-2]
730 destdir = os.path.join('repo', packageName, locale, dirname)
731 os.makedirs(destdir, mode=0o755, exist_ok=True)
732 sourcefile = os.path.join(root, f)
733 destfile = os.path.join(destdir, os.path.basename(f))
734 logging.debug('copying ' + sourcefile + ' ' + destfile)
735 shutil.copy(sourcefile, destfile)
738 def insert_localized_app_metadata(apps):
739 """scans standard locations for graphics and localized text
741 Scans for localized description files, store graphics, and
742 screenshot PNG files in statically defined screenshots directory
743 and adds them to the app metadata. The screenshots and graphic
744 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
745 and must be in the following layout:
746 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
748 repo/packageName/locale/featureGraphic.png
749 repo/packageName/locale/phoneScreenshots/1.png
750 repo/packageName/locale/phoneScreenshots/2.png
752 The changelog files must be text files named with the versionCode
753 ending with ".txt" and must be in the following layout:
754 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
756 repo/packageName/locale/changelogs/12345.txt
758 This will scan the each app's source repo then the metadata/ dir
759 for these standard locations of changelog files. If it finds
760 them, they will be added to the dict of all packages, with the
761 versions in the metadata/ folder taking precendence over the what
762 is in the app's source repo.
764 Where "packageName" is the app's packageName and "locale" is the locale
765 of the graphics, e.g. what language they are in, using the IETF RFC5646
766 format (en-US, fr-CA, es-MX, etc).
768 This will also scan the app's git for a fastlane folder, and the
769 metadata/ folder and the apps' source repos for standard locations
770 of graphic and screenshot files. If it finds them, it will copy
771 them into the repo. The fastlane files follow this pattern:
772 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
776 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
777 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
778 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
779 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
781 for srcd in sorted(sourcedirs):
782 if not os.path.isdir(srcd):
784 for root, dirs, files in os.walk(srcd):
785 segments = root.split('/')
786 packageName = segments[1]
787 if packageName not in apps:
788 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
790 locale = segments[-1]
791 destdir = os.path.join('repo', packageName, locale)
793 # flavours specified in build receipt
795 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
796 and 'gradle' in apps[packageName].builds[-1]:
797 build_flavours = apps[packageName].builds[-1].gradle
799 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
800 logging.debug("ignoring due to wrong flavour")
804 if f in ('description.txt', 'full_description.txt'):
805 _set_localized_text_entry(apps[packageName], locale, 'description',
806 os.path.join(root, f))
808 elif f in ('summary.txt', 'short_description.txt'):
809 _set_localized_text_entry(apps[packageName], locale, 'summary',
810 os.path.join(root, f))
812 elif f in ('name.txt', 'title.txt'):
813 _set_localized_text_entry(apps[packageName], locale, 'name',
814 os.path.join(root, f))
816 elif f == 'video.txt':
817 _set_localized_text_entry(apps[packageName], locale, 'video',
818 os.path.join(root, f))
820 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
821 locale = segments[-2]
822 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
823 os.path.join(root, f))
826 base, extension = common.get_extension(f)
827 if locale == 'images':
828 locale = segments[-2]
829 destdir = os.path.join('repo', packageName, locale)
830 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
831 os.makedirs(destdir, mode=0o755, exist_ok=True)
832 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
833 shutil.copy(os.path.join(root, f), destdir)
835 if d in SCREENSHOT_DIRS:
836 if locale == 'images':
837 locale = segments[-2]
838 destdir = os.path.join('repo', packageName, locale)
839 for f in glob.glob(os.path.join(root, d, '*.*')):
840 _ignored, extension = common.get_extension(f)
841 if extension in ALLOWED_EXTENSIONS:
842 screenshotdestdir = os.path.join(destdir, d)
843 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
844 logging.debug('copying ' + f + ' ' + screenshotdestdir)
845 shutil.copy(f, screenshotdestdir)
847 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
849 if not os.path.isdir(d):
851 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
852 if not os.path.isfile(f):
854 segments = f.split('/')
855 packageName = segments[1]
857 screenshotdir = segments[3]
858 filename = os.path.basename(f)
859 base, extension = common.get_extension(filename)
861 if packageName not in apps:
862 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
863 .format(path=filename, name=packageName))
865 graphics = _get_localized_dict(apps[packageName], locale)
867 if extension not in ALLOWED_EXTENSIONS:
868 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
869 elif base in GRAPHIC_NAMES:
870 # there can only be zero or one of these per locale
871 graphics[base] = filename
872 elif screenshotdir in SCREENSHOT_DIRS:
873 # there can any number of these per locale
874 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
875 if screenshotdir not in graphics:
876 graphics[screenshotdir] = []
877 graphics[screenshotdir].append(filename)
879 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
882 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
883 """Scan a repo for all files with an extension except APK/OBB
885 :param apkcache: current cached info about all repo files
886 :param repodir: repo directory to scan
887 :param knownapks: list of all known files, as per metadata.read_metadata
888 :param use_date_from_file: use date from file (instead of current date)
889 for newly added files
894 repodir = repodir.encode('utf-8')
895 for name in os.listdir(repodir):
896 file_extension = common.get_file_extension(name)
897 if file_extension == 'apk' or file_extension == 'obb':
899 filename = os.path.join(repodir, name)
900 name_utf8 = name.decode('utf-8')
901 if filename.endswith(b'_src.tar.gz'):
902 logging.debug(_('skipping source tarball: {path}')
903 .format(path=filename.decode('utf-8')))
905 if not common.is_repo_file(filename):
907 stat = os.stat(filename)
908 if stat.st_size == 0:
909 raise FDroidException(_('{path} is zero size!')
910 .format(path=filename))
912 shasum = sha256sum(filename)
915 repo_file = apkcache[name]
916 # added time is cached as tuple but used here as datetime instance
917 if 'added' in repo_file:
918 a = repo_file['added']
919 if isinstance(a, datetime):
920 repo_file['added'] = a
922 repo_file['added'] = datetime(*a[:6])
923 if repo_file.get('hash') == shasum:
924 logging.debug(_("Reading {apkfilename} from cache")
925 .format(apkfilename=name_utf8))
928 logging.debug(_("Ignoring stale cache data for {apkfilename}")
929 .format(apkfilename=name_utf8))
932 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
933 repo_file = collections.OrderedDict()
934 repo_file['name'] = os.path.splitext(name_utf8)[0]
935 # TODO rename apkname globally to something more generic
936 repo_file['apkName'] = name_utf8
937 repo_file['hash'] = shasum
938 repo_file['hashType'] = 'sha256'
939 repo_file['versionCode'] = 0
940 repo_file['versionName'] = shasum
941 # the static ID is the SHA256 unless it is set in the metadata
942 repo_file['packageName'] = shasum
944 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
946 repo_file['packageName'] = m.group(1)
947 repo_file['versionCode'] = int(m.group(2))
948 srcfilename = name + b'_src.tar.gz'
949 if os.path.exists(os.path.join(repodir, srcfilename)):
950 repo_file['srcname'] = srcfilename.decode('utf-8')
951 repo_file['size'] = stat.st_size
953 apkcache[name] = repo_file
956 if use_date_from_file:
957 timestamp = stat.st_ctime
958 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
960 default_date_param = None
962 # Record in knownapks, getting the added date at the same time..
963 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
964 default_date=default_date_param)
966 repo_file['added'] = added
968 repo_files.append(repo_file)
970 return repo_files, cachechanged
973 def scan_apk(apk_file):
975 Scans an APK file and returns dictionary with metadata of the APK.
977 Attention: This does *not* verify that the APK signature is correct.
979 :param apk_file: The (ideally absolute) path to the APK file
980 :raises BuildException
981 :return A dict containing APK metadata
984 'hash': sha256sum(apk_file),
985 'hashType': 'sha256',
986 'uses-permission': [],
987 'uses-permission-sdk-23': [],
991 'antiFeatures': set(),
994 if SdkToolsPopen(['aapt', 'version'], output=False):
995 scan_apk_aapt(apk, apk_file)
997 scan_apk_androguard(apk, apk_file)
999 # Get the signature, or rather the signing key fingerprints
1000 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1001 apk['sig'] = getsig(apk_file)
1003 raise BuildException("Failed to get apk signature")
1004 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1006 if not apk.get('signer'):
1007 raise BuildException("Failed to get apk signing key fingerprint")
1009 # Get size of the APK
1010 apk['size'] = os.path.getsize(apk_file)
1012 if 'minSdkVersion' not in apk:
1013 logging.warning("No SDK version information found in {0}".format(apk_file))
1014 apk['minSdkVersion'] = 1
1016 # Check for known vulnerabilities
1017 if has_known_vulnerability(apk_file):
1018 apk['antiFeatures'].add('KnownVuln')
1023 def scan_apk_aapt(apk, apkfile):
1024 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1025 if p.returncode != 0:
1026 if options.delete_unknown:
1027 if os.path.exists(apkfile):
1028 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1031 logging.error("Could not find {0} to remove it".format(apkfile))
1033 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1034 raise BuildException(_("Invalid APK"))
1035 for line in p.output.splitlines():
1036 if line.startswith("package:"):
1038 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1039 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1040 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1041 except Exception as e:
1042 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1043 elif line.startswith("application:"):
1044 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1045 # Keep path to non-dpi icon in case we need it
1046 match = re.match(APK_ICON_PAT_NODPI, line)
1048 apk['icons_src']['-1'] = match.group(1)
1049 elif line.startswith("launchable-activity:"):
1050 # Only use launchable-activity as fallback to application
1052 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1053 if '-1' not in apk['icons_src']:
1054 match = re.match(APK_ICON_PAT_NODPI, line)
1056 apk['icons_src']['-1'] = match.group(1)
1057 elif line.startswith("application-icon-"):
1058 match = re.match(APK_ICON_PAT, line)
1060 density = match.group(1)
1061 path = match.group(2)
1062 apk['icons_src'][density] = path
1063 elif line.startswith("sdkVersion:"):
1064 m = re.match(APK_SDK_VERSION_PAT, line)
1066 logging.error(line.replace('sdkVersion:', '')
1067 + ' is not a valid minSdkVersion!')
1069 apk['minSdkVersion'] = m.group(1)
1070 # if target not set, default to min
1071 if 'targetSdkVersion' not in apk:
1072 apk['targetSdkVersion'] = m.group(1)
1073 elif line.startswith("targetSdkVersion:"):
1074 m = re.match(APK_SDK_VERSION_PAT, line)
1076 logging.error(line.replace('targetSdkVersion:', '')
1077 + ' is not a valid targetSdkVersion!')
1079 apk['targetSdkVersion'] = m.group(1)
1080 elif line.startswith("maxSdkVersion:"):
1081 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1082 elif line.startswith("native-code:"):
1083 apk['nativecode'] = []
1084 for arch in line[13:].split(' '):
1085 apk['nativecode'].append(arch[1:-1])
1086 elif line.startswith('uses-permission:'):
1087 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1088 if perm_match['maxSdkVersion']:
1089 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1090 permission = UsesPermission(
1092 perm_match['maxSdkVersion']
1095 apk['uses-permission'].append(permission)
1096 elif line.startswith('uses-permission-sdk-23:'):
1097 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1098 if perm_match['maxSdkVersion']:
1099 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1100 permission_sdk_23 = UsesPermissionSdk23(
1102 perm_match['maxSdkVersion']
1105 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1107 elif line.startswith('uses-feature:'):
1108 feature = re.match(APK_FEATURE_PAT, line).group(1)
1109 # Filter out this, it's only added with the latest SDK tools and
1110 # causes problems for lots of apps.
1111 if feature != "android.hardware.screen.portrait" \
1112 and feature != "android.hardware.screen.landscape":
1113 if feature.startswith("android.feature."):
1114 feature = feature[16:]
1115 apk['features'].add(feature)
1118 def scan_apk_androguard(apk, apkfile):
1120 from androguard.core.bytecodes.apk import APK
1121 apkobject = APK(apkfile)
1122 if apkobject.is_valid_APK():
1123 arsc = apkobject.get_android_resources()
1125 if options.delete_unknown:
1126 if os.path.exists(apkfile):
1127 logging.error(_("Failed to get apk information, deleting {path}")
1128 .format(path=apkfile))
1131 logging.error(_("Could not find {path} to remove it")
1132 .format(path=apkfile))
1134 logging.error(_("Failed to get apk information, skipping {path}")
1135 .format(path=apkfile))
1136 raise BuildException(_("Invalid APK"))
1138 raise FDroidException("androguard library is not installed and aapt not present")
1139 except FileNotFoundError:
1140 logging.error(_("Could not open apk file for analysis"))
1141 raise BuildException(_("Invalid APK"))
1143 apk['packageName'] = apkobject.get_package()
1144 apk['versionCode'] = int(apkobject.get_androidversion_code())
1145 apk['versionName'] = apkobject.get_androidversion_name()
1146 if apk['versionName'][0] == "@":
1147 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1148 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1149 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1150 apk['name'] = apkobject.get_app_name()
1152 if apkobject.get_max_sdk_version() is not None:
1153 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1154 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1155 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1157 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1158 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1160 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1162 for file in apkobject.get_files():
1163 d_re = density_re.match(file)
1165 folder = d_re.group(1).split('-')
1167 resolution = folder[1]
1170 density = screen_resolutions[resolution]
1171 apk['icons_src'][density] = d_re.group(0)
1173 if apk['icons_src'].get('-1') is None:
1174 apk['icons_src']['-1'] = apk['icons_src']['160']
1176 arch_re = re.compile("^lib/(.*)/.*$")
1177 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1179 apk['nativecode'] = []
1180 apk['nativecode'].extend(sorted(list(arch)))
1182 xml = apkobject.get_android_manifest_xml()
1184 for item in xml.getElementsByTagName('uses-permission'):
1185 name = str(item.getAttribute("android:name"))
1186 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1187 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1188 permission = UsesPermission(
1192 apk['uses-permission'].append(permission)
1194 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1195 name = str(item.getAttribute("android:name"))
1196 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1197 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1198 permission_sdk_23 = UsesPermissionSdk23(
1202 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1204 for item in xml.getElementsByTagName('uses-feature'):
1205 feature = str(item.getAttribute("android:name"))
1206 if feature != "android.hardware.screen.portrait" \
1207 and feature != "android.hardware.screen.landscape":
1208 if feature.startswith("android.feature."):
1209 feature = feature[16:]
1210 apk['features'].append(feature)
1213 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1214 allow_disabled_algorithms=False, archive_bad_sig=False):
1215 """Processes the apk with the given filename in the given repo directory.
1217 This also extracts the icons.
1219 :param apkcache: current apk cache information
1220 :param apkfilename: the filename of the apk to scan
1221 :param repodir: repo directory to scan
1222 :param knownapks: known apks info
1223 :param use_date_from_apk: use date from APK (instead of current date)
1224 for newly added APKs
1225 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1226 disabled algorithms in the signature (e.g. MD5)
1227 :param archive_bad_sig: move APKs with a bad signature to the archive
1228 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1229 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1233 apkfile = os.path.join(repodir, apkfilename)
1235 cachechanged = False
1237 if apkfilename in apkcache:
1238 apk = apkcache[apkfilename]
1239 if apk.get('hash') == sha256sum(apkfile):
1240 logging.debug(_("Reading {apkfilename} from cache")
1241 .format(apkfilename=apkfilename))
1244 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1245 .format(apkfilename=apkfilename))
1248 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1251 apk = scan_apk(apkfile)
1252 except BuildException:
1253 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1254 .format(apkfilename=apkfilename))
1255 return True, None, False
1257 # Check for debuggable apks...
1258 if common.isApkAndDebuggable(apkfile):
1259 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1261 if options.rename_apks:
1262 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1263 std_short_name = os.path.join(repodir, n)
1264 if apkfile != std_short_name:
1265 if os.path.exists(std_short_name):
1266 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1267 if apkfile != std_long_name:
1268 if os.path.exists(std_long_name):
1269 dupdir = os.path.join('duplicates', repodir)
1270 if not os.path.isdir(dupdir):
1271 os.makedirs(dupdir, exist_ok=True)
1272 dupfile = os.path.join('duplicates', std_long_name)
1273 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1274 os.rename(apkfile, dupfile)
1275 return True, None, False
1277 os.rename(apkfile, std_long_name)
1278 apkfile = std_long_name
1280 os.rename(apkfile, std_short_name)
1281 apkfile = std_short_name
1282 apkfilename = apkfile[len(repodir) + 1:]
1284 apk['apkName'] = apkfilename
1285 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1286 if os.path.exists(os.path.join(repodir, srcfilename)):
1287 apk['srcname'] = srcfilename
1289 # verify the jar signature is correct, allow deprecated
1290 # algorithms only if the APK is in the archive.
1292 if not common.verify_apk_signature(apkfile):
1293 if repodir == 'archive' or allow_disabled_algorithms:
1294 if common.verify_old_apk_signature(apkfile):
1295 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1303 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1304 .format(apkfilename=apkfilename))
1305 move_apk_between_sections(repodir, 'archive', apk)
1307 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1308 .format(apkfilename=apkfilename))
1309 return True, None, False
1311 apkzip = zipfile.ZipFile(apkfile, 'r')
1313 manifest = apkzip.getinfo('AndroidManifest.xml')
1314 if manifest.date_time[1] == 0: # month can't be zero
1315 logging.debug(_('AndroidManifest.xml has no date'))
1317 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1319 # extract icons from APK zip file
1320 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1322 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1324 apkzip.close() # ensure that APK zip file gets closed
1326 # resize existing icons for densities missing in the APK
1327 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1329 if use_date_from_apk and manifest.date_time[1] != 0:
1330 default_date_param = datetime(*manifest.date_time)
1332 default_date_param = None
1334 # Record in known apks, getting the added date at the same time..
1335 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1336 default_date=default_date_param)
1338 apk['added'] = added
1340 apkcache[apkfilename] = apk
1343 return False, apk, cachechanged
1346 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1347 """Processes the apks in the given repo directory.
1349 This also extracts the icons.
1351 :param apkcache: current apk cache information
1352 :param repodir: repo directory to scan
1353 :param knownapks: known apks info
1354 :param use_date_from_apk: use date from APK (instead of current date)
1355 for newly added APKs
1356 :returns: (apks, cachechanged) where apks is a list of apk information,
1357 and cachechanged is True if the apkcache got changed.
1360 cachechanged = False
1362 for icon_dir in get_all_icon_dirs(repodir):
1363 if os.path.exists(icon_dir):
1365 shutil.rmtree(icon_dir)
1366 os.makedirs(icon_dir)
1368 os.makedirs(icon_dir)
1371 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1372 apkfilename = apkfile[len(repodir) + 1:]
1373 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1374 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1375 use_date_from_apk, ada, True)
1379 cachechanged = cachechanged or cachethis
1381 return apks, cachechanged
1384 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1386 Extracts icons from the given APK zip in various densities,
1387 saves them into given repo directory
1388 and stores their names in the APK metadata dictionary.
1390 :param icon_filename: A string representing the icon's file name
1391 :param apk: A populated dictionary containing APK metadata.
1392 Needs to have 'icons_src' key
1393 :param apkzip: An opened zipfile.ZipFile of the APK file
1394 :param repo_dir: The directory of the APK's repository
1395 :return: A list of icon densities that are missing
1397 empty_densities = []
1398 for density in screen_densities:
1399 if density not in apk['icons_src']:
1400 empty_densities.append(density)
1402 icon_src = apk['icons_src'][density]
1403 icon_dir = get_icon_dir(repo_dir, density)
1404 icon_dest = os.path.join(icon_dir, icon_filename)
1406 # Extract the icon files per density
1407 if icon_src.endswith('.xml'):
1408 png = os.path.basename(icon_src)[:-4] + '.png'
1409 for f in apkzip.namelist():
1411 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1412 if m and screen_resolutions[m.group(2)] == density:
1414 if icon_src.endswith('.xml'):
1415 empty_densities.append(density)
1418 with open(icon_dest, 'wb') as f:
1419 f.write(get_icon_bytes(apkzip, icon_src))
1420 apk['icons'][density] = icon_filename
1421 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1422 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1423 del apk['icons_src'][density]
1424 empty_densities.append(density)
1426 if '-1' in apk['icons_src']:
1427 icon_src = apk['icons_src']['-1']
1428 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1429 with open(icon_path, 'wb') as f:
1430 f.write(get_icon_bytes(apkzip, icon_src))
1432 im = Image.open(icon_path)
1433 dpi = px_to_dpi(im.size[0])
1434 for density in screen_densities:
1435 if density in apk['icons']:
1437 if density == screen_densities[-1] or dpi >= int(density):
1438 apk['icons'][density] = icon_filename
1439 shutil.move(icon_path,
1440 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1441 empty_densities.remove(density)
1443 except Exception as e:
1444 logging.warning(_("Failed reading {path}: {error}")
1445 .format(path=icon_path, error=e))
1448 apk['icon'] = icon_filename
1450 return empty_densities
1453 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1455 Resize existing icons for densities missing in the APK to ensure all densities are available
1457 :param empty_densities: A list of icon densities that are missing
1458 :param icon_filename: A string representing the icon's file name
1459 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1460 :param repo_dir: The directory of the APK's repository
1462 # First try resizing down to not lose quality
1464 for density in screen_densities:
1465 if density not in empty_densities:
1466 last_density = density
1468 if last_density is None:
1470 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1472 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1473 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1476 fp = open(last_icon_path, 'rb')
1479 size = dpi_to_px(density)
1481 im.thumbnail((size, size), Image.ANTIALIAS)
1482 im.save(icon_path, "PNG")
1483 empty_densities.remove(density)
1484 except Exception as e:
1485 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1490 # Then just copy from the highest resolution available
1492 for density in reversed(screen_densities):
1493 if density not in empty_densities:
1494 last_density = density
1497 if last_density is None:
1501 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1502 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1504 empty_densities.remove(density)
1506 for density in screen_densities:
1507 icon_dir = get_icon_dir(repo_dir, density)
1508 icon_dest = os.path.join(icon_dir, icon_filename)
1509 resize_icon(icon_dest, density)
1511 # Copy from icons-mdpi to icons since mdpi is the baseline density
1512 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1513 if os.path.isfile(baseline):
1514 apk['icons']['0'] = icon_filename
1515 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1518 def apply_info_from_latest_apk(apps, apks):
1520 Some information from the apks needs to be applied up to the application level.
1521 When doing this, we use the info from the most recent version's apk.
1522 We deal with figuring out when the app was added and last updated at the same time.
1524 for appid, app in apps.items():
1525 bestver = UNSET_VERSION_CODE
1527 if apk['packageName'] == appid:
1528 if apk['versionCode'] > bestver:
1529 bestver = apk['versionCode']
1533 if not app.added or apk['added'] < app.added:
1534 app.added = apk['added']
1535 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1536 app.lastUpdated = apk['added']
1539 logging.debug("Don't know when " + appid + " was added")
1540 if not app.lastUpdated:
1541 logging.debug("Don't know when " + appid + " was last updated")
1543 if bestver == UNSET_VERSION_CODE:
1545 if app.Name is None:
1546 app.Name = app.AutoName or appid
1548 logging.debug("Application " + appid + " has no packages")
1550 if app.Name is None:
1551 app.Name = bestapk['name']
1552 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1553 if app.CurrentVersionCode is None:
1554 app.CurrentVersionCode = str(bestver)
1557 def make_categories_txt(repodir, categories):
1558 '''Write a category list in the repo to allow quick access'''
1560 for cat in sorted(categories):
1561 catdata += cat + '\n'
1562 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1566 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1568 def filter_apk_list_sorted(apk_list):
1570 for apk in apk_list:
1571 if apk['packageName'] == appid:
1574 # Sort the apk list by version code. First is highest/newest.
1575 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1577 for appid, app in apps.items():
1579 if app.ArchivePolicy:
1580 keepversions = int(app.ArchivePolicy[:-9])
1582 keepversions = defaultkeepversions
1584 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1585 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1587 current_app_apks = filter_apk_list_sorted(apks)
1588 if len(current_app_apks) > keepversions:
1589 # Move back the ones we don't want.
1590 for apk in current_app_apks[keepversions:]:
1591 move_apk_between_sections(repodir, archivedir, apk)
1592 archapks.append(apk)
1595 current_app_archapks = filter_apk_list_sorted(archapks)
1596 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1598 # Move forward the ones we want again, except DisableAlgorithm
1599 for apk in current_app_archapks:
1600 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1601 move_apk_between_sections(archivedir, repodir, apk)
1602 archapks.remove(apk)
1605 if kept == keepversions:
1609 def move_apk_between_sections(from_dir, to_dir, apk):
1610 """move an APK from repo to archive or vice versa"""
1612 def _move_file(from_dir, to_dir, filename, ignore_missing):
1613 from_path = os.path.join(from_dir, filename)
1614 if ignore_missing and not os.path.exists(from_path):
1616 to_path = os.path.join(to_dir, filename)
1617 if not os.path.exists(to_dir):
1619 shutil.move(from_path, to_path)
1621 if from_dir == to_dir:
1624 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1625 _move_file(from_dir, to_dir, apk['apkName'], False)
1626 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1627 for density in all_screen_densities:
1628 from_icon_dir = get_icon_dir(from_dir, density)
1629 to_icon_dir = get_icon_dir(to_dir, density)
1630 if density not in apk.get('icons', []):
1632 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1633 if 'srcname' in apk:
1634 _move_file(from_dir, to_dir, apk['srcname'], False)
1637 def add_apks_to_per_app_repos(repodir, apks):
1638 apks_per_app = dict()
1640 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1641 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1642 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1643 apks_per_app[apk['packageName']] = apk
1645 if not os.path.exists(apk['per_app_icons']):
1646 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1647 os.makedirs(apk['per_app_icons'])
1649 apkpath = os.path.join(repodir, apk['apkName'])
1650 shutil.copy(apkpath, apk['per_app_repo'])
1651 apksigpath = apkpath + '.sig'
1652 if os.path.exists(apksigpath):
1653 shutil.copy(apksigpath, apk['per_app_repo'])
1654 apkascpath = apkpath + '.asc'
1655 if os.path.exists(apkascpath):
1656 shutil.copy(apkascpath, apk['per_app_repo'])
1659 def create_metadata_from_template(apk):
1660 '''create a new metadata file using internal or external template
1662 Generate warnings for apk's with no metadata (or create skeleton
1663 metadata files, if requested on the command line). Though the
1664 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1665 since those impose things on the metadata file made from the
1666 template: field sort order, empty field value, formatting, etc.
1670 if os.path.exists('template.yml'):
1671 with open('template.yml') as f:
1673 if 'name' in apk and apk['name'] != '':
1674 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1675 r'\1 ' + apk['name'],
1677 flags=re.IGNORECASE | re.MULTILINE)
1679 logging.warning(_('{appid} does not have a name! Using package name instead.')
1680 .format(appid=apk['packageName']))
1681 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1682 r'\1 ' + apk['packageName'],
1684 flags=re.IGNORECASE | re.MULTILINE)
1685 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1689 app['Categories'] = [os.path.basename(os.getcwd())]
1690 # include some blanks as part of the template
1691 app['AuthorName'] = ''
1694 app['IssueTracker'] = ''
1695 app['SourceCode'] = ''
1696 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1697 if 'name' in apk and apk['name'] != '':
1698 app['Name'] = apk['name']
1700 logging.warning(_('{appid} does not have a name! Using package name instead.')
1701 .format(appid=apk['packageName']))
1702 app['Name'] = apk['packageName']
1703 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1704 yaml.dump(app, f, default_flow_style=False)
1705 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1714 global config, options
1716 # Parse command line...
1717 parser = ArgumentParser()
1718 common.setup_global_opts(parser)
1719 parser.add_argument("--create-key", action="store_true", default=False,
1720 help=_("Add a repo signing key to an unsigned repo"))
1721 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1722 help=_("Add skeleton metadata files for APKs that are missing them"))
1723 parser.add_argument("--delete-unknown", action="store_true", default=False,
1724 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1725 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1726 help=_("Report on build data status"))
1727 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1728 help=_("Interactively ask about things that need updating."))
1729 parser.add_argument("-I", "--icons", action="store_true", default=False,
1730 help=_("Resize all the icons exceeding the max pixel size and exit"))
1731 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1732 help=_("Specify editor to use in interactive mode. Default " +
1733 "is {path}").format(path='/etc/alternatives/editor'))
1734 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1735 help=_("Update the wiki"))
1736 parser.add_argument("--pretty", action="store_true", default=False,
1737 help=_("Produce human-readable XML/JSON for index files"))
1738 parser.add_argument("--clean", action="store_true", default=False,
1739 help=_("Clean update - don't uses caches, reprocess all APKs"))
1740 parser.add_argument("--nosign", action="store_true", default=False,
1741 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1742 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1743 help=_("Use date from APK instead of current time for newly added APKs"))
1744 parser.add_argument("--rename-apks", action="store_true", default=False,
1745 help=_("Rename APK files that do not match package.name_123.apk"))
1746 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1747 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1748 metadata.add_metadata_arguments(parser)
1749 options = parser.parse_args()
1750 metadata.warnings_action = options.W
1752 config = common.read_config(options)
1754 if not ('jarsigner' in config and 'keytool' in config):
1755 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1758 if config['archive_older'] != 0:
1759 repodirs.append('archive')
1760 if not os.path.exists('archive'):
1764 resize_all_icons(repodirs)
1767 if options.rename_apks:
1768 options.clean = True
1770 # check that icons exist now, rather than fail at the end of `fdroid update`
1771 for k in ['repo_icon', 'archive_icon']:
1773 if not os.path.exists(config[k]):
1774 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1775 .format(name=k, path=config[k]))
1778 # if the user asks to create a keystore, do it now, reusing whatever it can
1779 if options.create_key:
1780 if os.path.exists(config['keystore']):
1781 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1782 logging.critical("\t'" + config['keystore'] + "'")
1785 if 'repo_keyalias' not in config:
1786 config['repo_keyalias'] = socket.getfqdn()
1787 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1788 if 'keydname' not in config:
1789 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1790 common.write_to_config(config, 'keydname', config['keydname'])
1791 if 'keystore' not in config:
1792 config['keystore'] = common.default_config['keystore']
1793 common.write_to_config(config, 'keystore', config['keystore'])
1795 password = common.genpassword()
1796 if 'keystorepass' not in config:
1797 config['keystorepass'] = password
1798 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1799 if 'keypass' not in config:
1800 config['keypass'] = password
1801 common.write_to_config(config, 'keypass', config['keypass'])
1802 common.genkeystore(config)
1805 apps = metadata.read_metadata()
1807 # Generate a list of categories...
1809 for app in apps.values():
1810 categories.update(app.Categories)
1812 # Read known apks data (will be updated and written back when we've finished)
1813 knownapks = common.KnownApks()
1816 apkcache = get_cache()
1818 # Delete builds for disabled apps
1819 delete_disabled_builds(apps, apkcache, repodirs)
1821 # Scan all apks in the main repo
1822 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1824 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1825 options.use_date_from_apk)
1826 cachechanged = cachechanged or fcachechanged
1829 if apk['packageName'] not in apps:
1830 if options.create_metadata:
1831 create_metadata_from_template(apk)
1832 apps = metadata.read_metadata()
1834 msg = _("{apkfilename} ({appid}) has no metadata!") \
1835 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1836 if options.delete_unknown:
1837 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1838 .format(apkfilename=apk['apkName']))
1839 rmf = os.path.join(repodirs[0], apk['apkName'])
1840 if not os.path.exists(rmf):
1841 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1845 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1847 copy_triple_t_store_metadata(apps)
1848 insert_obbs(repodirs[0], apps, apks)
1849 insert_localized_app_metadata(apps)
1850 translate_per_build_anti_features(apps, apks)
1852 # Scan the archive repo for apks as well
1853 if len(repodirs) > 1:
1854 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1860 # Apply information from latest apks to the application and update dates
1861 apply_info_from_latest_apk(apps, apks + archapks)
1863 # Sort the app list by name, then the web site doesn't have to by default.
1864 # (we had to wait until we'd scanned the apks to do this, because mostly the
1865 # name comes from there!)
1866 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1868 # APKs are placed into multiple repos based on the app package, providing
1869 # per-app subscription feeds for nightly builds and things like it
1870 if config['per_app_repos']:
1871 add_apks_to_per_app_repos(repodirs[0], apks)
1872 for appid, app in apps.items():
1873 repodir = os.path.join(appid, 'fdroid', 'repo')
1875 appdict[appid] = app
1876 if os.path.isdir(repodir):
1877 index.make(appdict, [appid], apks, repodir, False)
1879 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1882 if len(repodirs) > 1:
1883 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1885 # Make the index for the main repo...
1886 index.make(apps, sortedids, apks, repodirs[0], False)
1887 make_categories_txt(repodirs[0], categories)
1889 # If there's an archive repo, make the index for it. We already scanned it
1891 if len(repodirs) > 1:
1892 index.make(apps, sortedids, archapks, repodirs[1], True)
1894 git_remote = config.get('binary_transparency_remote')
1895 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1897 btlog.make_binary_transparency_log(repodirs)
1899 if config['update_stats']:
1900 # Update known apks info...
1901 knownapks.writeifchanged()
1903 # Generate latest apps data for widget
1904 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1906 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1908 appid = line.rstrip()
1909 data += appid + "\t"
1911 data += app.Name + "\t"
1912 if app.icon is not None:
1913 data += app.icon + "\t"
1914 data += app.License + "\n"
1915 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1919 write_cache(apkcache)
1921 # Update the wiki...
1923 update_wiki(apps, sortedids, apks + archapks)
1925 logging.info(_("Finished"))
1928 if __name__ == "__main__":