3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
76 all_screen_densities = ['0'] + screen_densities
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 def dpi_to_px(density):
88 return (int(density) * 48) / 160
92 return (int(px) * 160) / 48
95 def get_icon_dir(repodir, density):
97 return os.path.join(repodir, "icons")
98 return os.path.join(repodir, "icons-%s" % density)
101 def get_icon_dirs(repodir):
102 for density in screen_densities:
103 yield get_icon_dir(repodir, density)
106 def get_all_icon_dirs(repodir):
107 for density in all_screen_densities:
108 yield get_icon_dir(repodir, density)
111 def update_wiki(apps, sortedids, apks):
114 :param apps: fully populated list of all applications
115 :param apks: all apks, except...
117 logging.info("Updating wiki")
119 wikiredircat = 'App Redirects'
121 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122 path=config['wiki_path'])
123 site.login(config['wiki_user'], config['wiki_password'])
125 generated_redirects = {}
127 for appid in sortedids:
128 app = metadata.App(apps[appid])
132 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
134 for af in app.AntiFeatures:
135 wikidata += '{{AntiFeature|' + af + '}}\n'
140 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
143 app.added.strftime('%Y-%m-%d') if app.added else '',
144 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
159 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
161 wikidata += app.Summary
162 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
164 wikidata += "=Description=\n"
165 wikidata += metadata.description_wiki(app.Description) + "\n"
167 wikidata += "=Maintainer Notes=\n"
168 if app.MaintainerNotes:
169 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
172 # Get a list of all packages for this application...
174 gotcurrentver = False
178 if apk['packageName'] == appid:
179 if str(apk['versionCode']) == app.CurrentVersionCode:
182 # Include ones we can't build, as a special case...
183 for build in app.builds:
185 if build.versionCode == app.CurrentVersionCode:
187 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188 apklist.append({'versionCode': int(build.versionCode),
189 'versionName': build.versionName,
190 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
195 if apk['versionCode'] == int(build.versionCode):
200 apklist.append({'versionCode': int(build.versionCode),
201 'versionName': build.versionName,
202 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
204 if app.CurrentVersionCode == '0':
206 # Sort with most recent first...
207 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
209 wikidata += "=Versions=\n"
210 if len(apklist) == 0:
211 wikidata += "We currently have no versions of this app available."
212 elif not gotcurrentver:
213 wikidata += "We don't have the current version of this app."
215 wikidata += "We have the current version of this app."
216 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218 if len(app.NoSourceSince) > 0:
219 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220 if len(app.CurrentVersion) > 0:
221 wikidata += "The current (recommended) version is " + app.CurrentVersion
222 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
225 wikidata += "==" + apk['versionName'] + "==\n"
227 if 'buildproblem' in apk:
228 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
231 wikidata += "This version is built and signed by "
233 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
235 wikidata += "the original developer.\n\n"
236 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
238 wikidata += '\n[[Category:' + wikicat + ']]\n'
239 if len(app.NoSourceSince) > 0:
240 wikidata += '\n[[Category:Apps missing source code]]\n'
241 if validapks == 0 and not app.Disabled:
242 wikidata += '\n[[Category:Apps with no packages]]\n'
243 if cantupdate and not app.Disabled:
244 wikidata += "\n[[Category:Apps we cannot update]]\n"
245 if buildfails and not app.Disabled:
246 wikidata += "\n[[Category:Apps with failing builds]]\n"
247 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248 wikidata += '\n[[Category:Apps to Update]]\n'
250 wikidata += '\n[[Category:Apps that are disabled]]\n'
251 if app.UpdateCheckMode == 'None' and not app.Disabled:
252 wikidata += '\n[[Category:Apps with no update check]]\n'
253 for appcat in app.Categories:
254 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
256 # We can't have underscores in the page name, even if they're in
257 # the package ID, because MediaWiki messes with them...
258 pagename = appid.replace('_', ' ')
260 # Drop a trailing newline, because mediawiki is going to drop it anyway
261 # and it we don't we'll think the page has changed when it hasn't...
262 if wikidata.endswith('\n'):
263 wikidata = wikidata[:-1]
265 generated_pages[pagename] = wikidata
267 # Make a redirect from the name to the ID too, unless there's
268 # already an existing page with the name and it isn't a redirect.
270 apppagename = app.Name.replace('_', ' ')
271 apppagename = apppagename.replace('{', '')
272 apppagename = apppagename.replace('}', ' ')
273 apppagename = apppagename.replace(':', ' ')
274 apppagename = apppagename.replace('[', ' ')
275 apppagename = apppagename.replace(']', ' ')
276 # Drop double spaces caused mostly by replacing ':' above
277 apppagename = apppagename.replace(' ', ' ')
278 for expagename in site.allpages(prefix=apppagename,
279 filterredir='nonredirects',
281 if expagename == apppagename:
283 # Another reason not to make the redirect page is if the app name
284 # is the same as it's ID, because that will overwrite the real page
285 # with an redirect to itself! (Although it seems like an odd
286 # scenario this happens a lot, e.g. where there is metadata but no
287 # builds or binaries to extract a name from.
288 if apppagename == pagename:
291 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
293 for tcat, genp in [(wikicat, generated_pages),
294 (wikiredircat, generated_redirects)]:
295 catpages = site.Pages['Category:' + tcat]
297 for page in catpages:
298 existingpages.append(page.name)
299 if page.name in genp:
300 pagetxt = page.edit()
301 if pagetxt != genp[page.name]:
302 logging.debug("Updating modified page " + page.name)
303 page.save(genp[page.name], summary='Auto-updated')
305 logging.debug("Page " + page.name + " is unchanged")
307 logging.warn("Deleting page " + page.name)
308 page.delete('No longer published')
309 for pagename, text in genp.items():
310 logging.debug("Checking " + pagename)
311 if pagename not in existingpages:
312 logging.debug("Creating page " + pagename)
314 newpage = site.Pages[pagename]
315 newpage.save(text, summary='Auto-created')
316 except Exception as e:
317 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
319 # Purge server cache to ensure counts are up to date
320 site.pages['Repository Maintenance'].purge()
323 def delete_disabled_builds(apps, apkcache, repodirs):
324 """Delete disabled build outputs.
326 :param apps: list of all applications, as per metadata.read_metadata
327 :param apkcache: current apk cache information
328 :param repodirs: the repo directories to process
330 for appid, app in apps.items():
331 for build in app['builds']:
332 if not build.disable:
334 apkfilename = common.get_release_filename(app, build)
335 iconfilename = "%s.%s.png" % (
338 for repodir in repodirs:
340 os.path.join(repodir, apkfilename),
341 os.path.join(repodir, apkfilename + '.asc'),
342 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
344 for density in all_screen_densities:
345 repo_dir = get_icon_dir(repodir, density)
346 files.append(os.path.join(repo_dir, iconfilename))
349 if os.path.exists(f):
350 logging.info("Deleting disabled build output " + f)
352 if apkfilename in apkcache:
353 del apkcache[apkfilename]
356 def resize_icon(iconpath, density):
358 if not os.path.isfile(iconpath):
363 fp = open(iconpath, 'rb')
365 size = dpi_to_px(density)
367 if any(length > size for length in im.size):
369 im.thumbnail((size, size), Image.ANTIALIAS)
370 logging.debug("%s was too large at %s - new size is %s" % (
371 iconpath, oldsize, im.size))
372 im.save(iconpath, "PNG")
374 except Exception as e:
375 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
382 def resize_all_icons(repodirs):
383 """Resize all icons that exceed the max size
385 :param repodirs: the repo directories to process
387 for repodir in repodirs:
388 for density in screen_densities:
389 icon_dir = get_icon_dir(repodir, density)
390 icon_glob = os.path.join(icon_dir, '*.png')
391 for iconpath in glob.glob(icon_glob):
392 resize_icon(iconpath, density)
396 """ Get the signing certificate of an apk. To get the same md5 has that
397 Android gets, we encode the .RSA certificate in a specific format and pass
398 it hex-encoded to the md5 digest algorithm.
400 :param apkpath: path to the apk
401 :returns: A string containing the md5 of the signature of the apk or None
402 if an error occurred.
405 with zipfile.ZipFile(apkpath, 'r') as apk:
406 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
409 logging.error("Found no signing certificates on %s" % apkpath)
412 logging.error("Found multiple signing certificates on %s" % apkpath)
415 cert = apk.read(certs[0])
417 cert_encoded = common.get_certificate(cert)
419 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
422 def get_cache_file():
423 return os.path.join('tmp', 'apkcache')
427 """Get the cached dict of the APK index
429 Gather information about all the apk files in the repo directory,
430 using cached data if possible. Some of the index operations take a
431 long time, like calculating the SHA-256 and verifying the APK
434 The cache is invalidated if the metadata version is different, or
435 the 'allow_disabled_algorithms' config/option is different. In
436 those cases, there is no easy way to know what has changed from
437 the cache, so just rerun the whole thing.
442 apkcachefile = get_cache_file()
443 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
444 if not options.clean and os.path.exists(apkcachefile):
445 with open(apkcachefile, 'rb') as cf:
446 apkcache = pickle.load(cf, encoding='utf-8')
447 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
448 or apkcache.get('allow_disabled_algorithms') != ada:
453 apkcache["METADATA_VERSION"] = METADATA_VERSION
454 apkcache['allow_disabled_algorithms'] = ada
459 def write_cache(apkcache):
460 apkcachefile = get_cache_file()
461 cache_path = os.path.dirname(apkcachefile)
462 if not os.path.exists(cache_path):
463 os.makedirs(cache_path)
464 with open(apkcachefile, 'wb') as cf:
465 pickle.dump(apkcache, cf)
468 def get_icon_bytes(apkzip, iconsrc):
469 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
471 return apkzip.read(iconsrc)
473 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
476 def sha256sum(filename):
477 '''Calculate the sha256 of the given file'''
478 sha = hashlib.sha256()
479 with open(filename, 'rb') as f:
485 return sha.hexdigest()
488 def has_known_vulnerability(filename):
489 """checks for known vulnerabilities in the APK
491 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
492 version. Google also enforces this:
493 https://support.google.com/faqs/answer/6376725?hl=en
495 Checks whether there are more than one classes.dex or AndroidManifest.xml
496 files, which is invalid and an essential part of the "Master Key" attack.
498 http://www.saurik.com/id/17
501 # statically load this pattern
502 if not hasattr(has_known_vulnerability, "pattern"):
503 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
506 with zipfile.ZipFile(filename) as zf:
507 for name in zf.namelist():
508 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
511 chunk = lib.read(4096)
514 m = has_known_vulnerability.pattern.search(chunk)
516 version = m.group(1).decode('ascii')
517 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
518 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
519 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
520 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
522 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
525 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
526 if name in files_in_apk:
528 files_in_apk.add(name)
533 def insert_obbs(repodir, apps, apks):
534 """Scans the .obb files in a given repo directory and adds them to the
535 relevant APK instances. OBB files have versionCodes like APK
536 files, and they are loosely associated. If there is an OBB file
537 present, then any APK with the same or higher versionCode will use
538 that OBB file. There are two OBB types: main and patch, each APK
539 can only have only have one of each.
541 https://developer.android.com/google/play/expansion-files.html
543 :param repodir: repo directory to scan
544 :param apps: list of current, valid apps
545 :param apks: current information on all APKs
549 def obbWarnDelete(f, msg):
550 logging.warning(msg + f)
551 if options.delete_unknown:
552 logging.error("Deleting unknown file: " + f)
556 java_Integer_MIN_VALUE = -pow(2, 31)
557 currentPackageNames = apps.keys()
558 for f in glob.glob(os.path.join(repodir, '*.obb')):
559 obbfile = os.path.basename(f)
560 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
561 chunks = obbfile.split('.')
562 if chunks[0] != 'main' and chunks[0] != 'patch':
563 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
565 if not re.match(r'^-?[0-9]+$', chunks[1]):
566 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
568 versionCode = int(chunks[1])
569 packagename = ".".join(chunks[2:-1])
571 highestVersionCode = java_Integer_MIN_VALUE
572 if packagename not in currentPackageNames:
573 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
576 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
577 highestVersionCode = apk['versionCode']
578 if versionCode > highestVersionCode:
579 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
580 + ') than any APK: ')
582 obbsha256 = sha256sum(f)
583 obbs.append((packagename, versionCode, obbfile, obbsha256))
586 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
587 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
588 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
589 apk['obbMainFile'] = obbfile
590 apk['obbMainFileSha256'] = obbsha256
591 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
592 apk['obbPatchFile'] = obbfile
593 apk['obbPatchFileSha256'] = obbsha256
594 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
598 def translate_per_build_anti_features(apps, apks):
599 """Grab the anti-features list from the build metadata
601 For most Anti-Features, they are really most applicable per-APK,
602 not for an app. An app can fix a vulnerability, add/remove
603 tracking, etc. This reads the 'antifeatures' list from the Build
604 entries in the fdroiddata metadata file, then transforms it into
605 the 'antiFeatures' list of unique items for the index.
607 The field key is all lower case in the metadata file to match the
608 rest of the Build fields. It is 'antiFeatures' camel case in the
609 implementation, index, and fdroidclient since it is translated
610 from the build 'antifeatures' field, not directly included.
614 antiFeatures = dict()
615 for packageName, app in apps.items():
617 for build in app['builds']:
618 afl = build.get('antifeatures')
620 d[int(build.versionCode)] = afl
622 antiFeatures[packageName] = d
625 d = antiFeatures.get(apk['packageName'])
627 afl = d.get(apk['versionCode'])
629 apk['antiFeatures'].update(afl)
632 def _get_localized_dict(app, locale):
633 '''get the dict to add localized store metadata to'''
634 if 'localized' not in app:
635 app['localized'] = collections.OrderedDict()
636 if locale not in app['localized']:
637 app['localized'][locale] = collections.OrderedDict()
638 return app['localized'][locale]
641 def _set_localized_text_entry(app, locale, key, f):
642 limit = config['char_limits'][key]
643 localized = _get_localized_dict(app, locale)
645 text = fp.read()[:limit]
647 localized[key] = text
650 def _set_author_entry(app, key, f):
651 limit = config['char_limits']['author']
653 text = fp.read()[:limit]
658 def copy_triple_t_store_metadata(apps):
659 """Include store metadata from the app's source repo
661 The Triple-T Gradle Play Publisher is a plugin that has a standard
662 file layout for all of the metadata and graphics that the Google
663 Play Store accepts. Since F-Droid has the git repo, it can just
664 pluck those files directly. This method reads any text files into
665 the app dict, then copies any graphics into the fdroid repo
668 This needs to be run before insert_localized_app_metadata() so that
669 the graphics files that are copied into the fdroid repo get
672 https://github.com/Triple-T/gradle-play-publisher#upload-images
673 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
677 if not os.path.isdir('build'):
678 return # nothing to do
680 for packageName, app in apps.items():
681 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
682 logging.debug('Triple-T Gradle Play Publisher: ' + d)
683 for root, dirs, files in os.walk(d):
684 segments = root.split('/')
685 locale = segments[-2]
687 if f == 'fulldescription':
688 _set_localized_text_entry(app, locale, 'description',
689 os.path.join(root, f))
691 elif f == 'shortdescription':
692 _set_localized_text_entry(app, locale, 'summary',
693 os.path.join(root, f))
696 _set_localized_text_entry(app, locale, 'name',
697 os.path.join(root, f))
700 _set_localized_text_entry(app, locale, 'video',
701 os.path.join(root, f))
703 elif f == 'whatsnew':
704 _set_localized_text_entry(app, segments[-1], 'whatsNew',
705 os.path.join(root, f))
707 elif f == 'contactEmail':
708 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
710 elif f == 'contactPhone':
711 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
713 elif f == 'contactWebsite':
714 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
717 base, extension = common.get_extension(f)
718 dirname = os.path.basename(root)
719 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
720 if segments[-2] == 'listing':
721 locale = segments[-3]
723 locale = segments[-2]
724 destdir = os.path.join('repo', packageName, locale)
725 os.makedirs(destdir, mode=0o755, exist_ok=True)
726 sourcefile = os.path.join(root, f)
727 destfile = os.path.join(destdir, dirname + '.' + extension)
728 logging.debug('copying ' + sourcefile + ' ' + destfile)
729 shutil.copy(sourcefile, destfile)
732 def insert_localized_app_metadata(apps):
733 """scans standard locations for graphics and localized text
735 Scans for localized description files, store graphics, and
736 screenshot PNG files in statically defined screenshots directory
737 and adds them to the app metadata. The screenshots and graphic
738 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
739 and must be in the following layout:
740 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
742 repo/packageName/locale/featureGraphic.png
743 repo/packageName/locale/phoneScreenshots/1.png
744 repo/packageName/locale/phoneScreenshots/2.png
746 The changelog files must be text files named with the versionCode
747 ending with ".txt" and must be in the following layout:
748 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
750 repo/packageName/locale/changelogs/12345.txt
752 This will scan the each app's source repo then the metadata/ dir
753 for these standard locations of changelog files. If it finds
754 them, they will be added to the dict of all packages, with the
755 versions in the metadata/ folder taking precendence over the what
756 is in the app's source repo.
758 Where "packageName" is the app's packageName and "locale" is the locale
759 of the graphics, e.g. what language they are in, using the IETF RFC5646
760 format (en-US, fr-CA, es-MX, etc).
762 This will also scan the app's git for a fastlane folder, and the
763 metadata/ folder and the apps' source repos for standard locations
764 of graphic and screenshot files. If it finds them, it will copy
765 them into the repo. The fastlane files follow this pattern:
766 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
770 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
771 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
772 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
774 for srcd in sorted(sourcedirs):
775 if not os.path.isdir(srcd):
777 for root, dirs, files in os.walk(srcd):
778 segments = root.split('/')
779 packageName = segments[1]
780 if packageName not in apps:
781 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
783 locale = segments[-1]
784 destdir = os.path.join('repo', packageName, locale)
786 if f in ('description.txt', 'full_description.txt'):
787 _set_localized_text_entry(apps[packageName], locale, 'description',
788 os.path.join(root, f))
790 elif f in ('summary.txt', 'short_description.txt'):
791 _set_localized_text_entry(apps[packageName], locale, 'summary',
792 os.path.join(root, f))
794 elif f in ('name.txt', 'title.txt'):
795 _set_localized_text_entry(apps[packageName], locale, 'name',
796 os.path.join(root, f))
798 elif f == 'video.txt':
799 _set_localized_text_entry(apps[packageName], locale, 'video',
800 os.path.join(root, f))
802 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
803 locale = segments[-2]
804 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
805 os.path.join(root, f))
808 base, extension = common.get_extension(f)
809 if locale == 'images':
810 locale = segments[-2]
811 destdir = os.path.join('repo', packageName, locale)
812 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
813 os.makedirs(destdir, mode=0o755, exist_ok=True)
814 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
815 shutil.copy(os.path.join(root, f), destdir)
817 if d in SCREENSHOT_DIRS:
818 for f in glob.glob(os.path.join(root, d, '*.*')):
819 _, extension = common.get_extension(f)
820 if extension in ALLOWED_EXTENSIONS:
821 screenshotdestdir = os.path.join(destdir, d)
822 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
823 logging.debug('copying ' + f + ' ' + screenshotdestdir)
824 shutil.copy(f, screenshotdestdir)
826 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
828 if not os.path.isdir(d):
830 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
831 if not os.path.isfile(f):
833 segments = f.split('/')
834 packageName = segments[1]
836 screenshotdir = segments[3]
837 filename = os.path.basename(f)
838 base, extension = common.get_extension(filename)
840 if packageName not in apps:
841 logging.warning('Found "%s" graphic without metadata for app "%s"!'
842 % (filename, packageName))
844 graphics = _get_localized_dict(apps[packageName], locale)
846 if extension not in ALLOWED_EXTENSIONS:
847 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
848 elif base in GRAPHIC_NAMES:
849 # there can only be zero or one of these per locale
850 graphics[base] = filename
851 elif screenshotdir in SCREENSHOT_DIRS:
852 # there can any number of these per locale
853 logging.debug('adding to ' + screenshotdir + ': ' + f)
854 if screenshotdir not in graphics:
855 graphics[screenshotdir] = []
856 graphics[screenshotdir].append(filename)
858 logging.warning('Unsupported graphics file found: ' + f)
861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
862 """Scan a repo for all files with an extension except APK/OBB
864 :param apkcache: current cached info about all repo files
865 :param repodir: repo directory to scan
866 :param knownapks: list of all known files, as per metadata.read_metadata
867 :param use_date_from_file: use date from file (instead of current date)
868 for newly added files
873 repodir = repodir.encode('utf-8')
874 for name in os.listdir(repodir):
875 file_extension = common.get_file_extension(name)
876 if file_extension == 'apk' or file_extension == 'obb':
878 filename = os.path.join(repodir, name)
879 name_utf8 = name.decode('utf-8')
880 if filename.endswith(b'_src.tar.gz'):
881 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
883 if not common.is_repo_file(filename):
885 stat = os.stat(filename)
886 if stat.st_size == 0:
887 raise FDroidException(filename + ' is zero size!')
889 shasum = sha256sum(filename)
892 repo_file = apkcache[name]
893 # added time is cached as tuple but used here as datetime instance
894 if 'added' in repo_file:
895 a = repo_file['added']
896 if isinstance(a, datetime):
897 repo_file['added'] = a
899 repo_file['added'] = datetime(*a[:6])
900 if repo_file.get('hash') == shasum:
901 logging.debug("Reading " + name_utf8 + " from cache")
904 logging.debug("Ignoring stale cache data for " + name)
907 logging.debug("Processing " + name_utf8)
908 repo_file = collections.OrderedDict()
909 repo_file['name'] = os.path.splitext(name_utf8)[0]
910 # TODO rename apkname globally to something more generic
911 repo_file['apkName'] = name_utf8
912 repo_file['hash'] = shasum
913 repo_file['hashType'] = 'sha256'
914 repo_file['versionCode'] = 0
915 repo_file['versionName'] = shasum
916 # the static ID is the SHA256 unless it is set in the metadata
917 repo_file['packageName'] = shasum
919 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
921 repo_file['packageName'] = m.group(1)
922 repo_file['versionCode'] = int(m.group(2))
923 srcfilename = name + b'_src.tar.gz'
924 if os.path.exists(os.path.join(repodir, srcfilename)):
925 repo_file['srcname'] = srcfilename.decode('utf-8')
926 repo_file['size'] = stat.st_size
928 apkcache[name] = repo_file
931 if use_date_from_file:
932 timestamp = stat.st_ctime
933 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
935 default_date_param = None
937 # Record in knownapks, getting the added date at the same time..
938 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
939 default_date=default_date_param)
941 repo_file['added'] = added
943 repo_files.append(repo_file)
945 return repo_files, cachechanged
948 def scan_apk(apk_file):
950 Scans an APK file and returns dictionary with metadata of the APK.
952 Attention: This does *not* verify that the APK signature is correct.
954 :param apk_file: The (ideally absolute) path to the APK file
955 :raises BuildException
956 :return A dict containing APK metadata
959 'hash': sha256sum(apk_file),
960 'hashType': 'sha256',
961 'uses-permission': [],
962 'uses-permission-sdk-23': [],
966 'antiFeatures': set(),
969 if SdkToolsPopen(['aapt', 'version'], output=False):
970 scan_apk_aapt(apk, apk_file)
972 scan_apk_androguard(apk, apk_file)
975 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
976 apk['sig'] = getsig(apk_file)
978 raise BuildException("Failed to get apk signature")
980 # Get size of the APK
981 apk['size'] = os.path.getsize(apk_file)
983 if 'minSdkVersion' not in apk:
984 logging.warning("No SDK version information found in {0}".format(apk_file))
985 apk['minSdkVersion'] = 1
987 # Check for known vulnerabilities
988 if has_known_vulnerability(apk_file):
989 apk['antiFeatures'].add('KnownVuln')
994 def scan_apk_aapt(apk, apkfile):
995 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
996 if p.returncode != 0:
997 if options.delete_unknown:
998 if os.path.exists(apkfile):
999 logging.error("Failed to get apk information, deleting " + apkfile)
1002 logging.error("Could not find {0} to remove it".format(apkfile))
1004 logging.error("Failed to get apk information, skipping " + apkfile)
1005 raise BuildException("Invalid APK")
1006 for line in p.output.splitlines():
1007 if line.startswith("package:"):
1009 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1010 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1011 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1012 except Exception as e:
1013 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1014 elif line.startswith("application:"):
1015 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1016 # Keep path to non-dpi icon in case we need it
1017 match = re.match(APK_ICON_PAT_NODPI, line)
1019 apk['icons_src']['-1'] = match.group(1)
1020 elif line.startswith("launchable-activity:"):
1021 # Only use launchable-activity as fallback to application
1023 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1024 if '-1' not in apk['icons_src']:
1025 match = re.match(APK_ICON_PAT_NODPI, line)
1027 apk['icons_src']['-1'] = match.group(1)
1028 elif line.startswith("application-icon-"):
1029 match = re.match(APK_ICON_PAT, line)
1031 density = match.group(1)
1032 path = match.group(2)
1033 apk['icons_src'][density] = path
1034 elif line.startswith("sdkVersion:"):
1035 m = re.match(APK_SDK_VERSION_PAT, line)
1037 logging.error(line.replace('sdkVersion:', '')
1038 + ' is not a valid minSdkVersion!')
1040 apk['minSdkVersion'] = m.group(1)
1041 # if target not set, default to min
1042 if 'targetSdkVersion' not in apk:
1043 apk['targetSdkVersion'] = m.group(1)
1044 elif line.startswith("targetSdkVersion:"):
1045 m = re.match(APK_SDK_VERSION_PAT, line)
1047 logging.error(line.replace('targetSdkVersion:', '')
1048 + ' is not a valid targetSdkVersion!')
1050 apk['targetSdkVersion'] = m.group(1)
1051 elif line.startswith("maxSdkVersion:"):
1052 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1053 elif line.startswith("native-code:"):
1054 apk['nativecode'] = []
1055 for arch in line[13:].split(' '):
1056 apk['nativecode'].append(arch[1:-1])
1057 elif line.startswith('uses-permission:'):
1058 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1059 if perm_match['maxSdkVersion']:
1060 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1061 permission = UsesPermission(
1063 perm_match['maxSdkVersion']
1066 apk['uses-permission'].append(permission)
1067 elif line.startswith('uses-permission-sdk-23:'):
1068 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1069 if perm_match['maxSdkVersion']:
1070 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1071 permission_sdk_23 = UsesPermissionSdk23(
1073 perm_match['maxSdkVersion']
1076 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1078 elif line.startswith('uses-feature:'):
1079 feature = re.match(APK_FEATURE_PAT, line).group(1)
1080 # Filter out this, it's only added with the latest SDK tools and
1081 # causes problems for lots of apps.
1082 if feature != "android.hardware.screen.portrait" \
1083 and feature != "android.hardware.screen.landscape":
1084 if feature.startswith("android.feature."):
1085 feature = feature[16:]
1086 apk['features'].add(feature)
1089 def scan_apk_androguard(apk, apkfile):
1091 from androguard.core.bytecodes.apk import APK
1092 apkobject = APK(apkfile)
1093 if apkobject.is_valid_APK():
1094 arsc = apkobject.get_android_resources()
1096 if options.delete_unknown:
1097 if os.path.exists(apkfile):
1098 logging.error("Failed to get apk information, deleting " + apkfile)
1101 logging.error("Could not find {0} to remove it".format(apkfile))
1103 logging.error("Failed to get apk information, skipping " + apkfile)
1104 raise BuildException("Invaild APK")
1106 raise FDroidException("androguard library is not installed and aapt not present")
1107 except FileNotFoundError:
1108 logging.error("Could not open apk file for analysis")
1109 raise BuildException("Invalid APK")
1111 apk['packageName'] = apkobject.get_package()
1112 apk['versionCode'] = int(apkobject.get_androidversion_code())
1113 apk['versionName'] = apkobject.get_androidversion_name()
1114 if apk['versionName'][0] == "@":
1115 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1116 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1117 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1118 apk['name'] = apkobject.get_app_name()
1120 if apkobject.get_max_sdk_version() is not None:
1121 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1122 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1123 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1125 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1126 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1128 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1130 for file in apkobject.get_files():
1131 d_re = density_re.match(file)
1133 folder = d_re.group(1).split('-')
1135 resolution = folder[1]
1138 density = screen_resolutions[resolution]
1139 apk['icons_src'][density] = d_re.group(0)
1141 if apk['icons_src'].get('-1') is None:
1142 apk['icons_src']['-1'] = apk['icons_src']['160']
1144 arch_re = re.compile("^lib/(.*)/.*$")
1145 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1147 apk['nativecode'] = []
1148 apk['nativecode'].extend(sorted(list(arch)))
1150 xml = apkobject.get_android_manifest_xml()
1152 for item in xml.getElementsByTagName('uses-permission'):
1153 name = str(item.getAttribute("android:name"))
1154 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1155 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1156 permission = UsesPermission(
1160 apk['uses-permission'].append(permission)
1162 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1163 name = str(item.getAttribute("android:name"))
1164 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1165 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1166 permission_sdk_23 = UsesPermissionSdk23(
1170 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1172 for item in xml.getElementsByTagName('uses-feature'):
1173 feature = str(item.getAttribute("android:name"))
1174 if feature != "android.hardware.screen.portrait" \
1175 and feature != "android.hardware.screen.landscape":
1176 if feature.startswith("android.feature."):
1177 feature = feature[16:]
1178 apk['features'].append(feature)
1181 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1182 allow_disabled_algorithms=False, archive_bad_sig=False):
1183 """Processes the apk with the given filename in the given repo directory.
1185 This also extracts the icons.
1187 :param apkcache: current apk cache information
1188 :param apkfilename: the filename of the apk to scan
1189 :param repodir: repo directory to scan
1190 :param knownapks: known apks info
1191 :param use_date_from_apk: use date from APK (instead of current date)
1192 for newly added APKs
1193 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1194 disabled algorithms in the signature (e.g. MD5)
1195 :param archive_bad_sig: move APKs with a bad signature to the archive
1196 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1197 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1200 if ' ' in apkfilename:
1201 if options.rename_apks:
1202 newfilename = apkfilename.replace(' ', '_')
1203 os.rename(os.path.join(repodir, apkfilename),
1204 os.path.join(repodir, newfilename))
1205 apkfilename = newfilename
1207 logging.critical("Spaces in filenames are not allowed.")
1208 return True, None, False
1211 apkfile = os.path.join(repodir, apkfilename)
1213 cachechanged = False
1215 if apkfilename in apkcache:
1216 apk = apkcache[apkfilename]
1217 if apk.get('hash') == sha256sum(apkfile):
1218 logging.debug("Reading " + apkfilename + " from cache")
1221 logging.debug("Ignoring stale cache data for " + apkfilename)
1224 logging.debug("Processing " + apkfilename)
1227 apk = scan_apk(apkfile)
1228 except BuildException:
1229 logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1230 return True, None, False
1232 # Check for debuggable apks...
1233 if common.isApkAndDebuggable(apkfile):
1234 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1236 if options.rename_apks:
1237 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1238 std_short_name = os.path.join(repodir, n)
1239 if apkfile != std_short_name:
1240 if os.path.exists(std_short_name):
1241 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1242 if apkfile != std_long_name:
1243 if os.path.exists(std_long_name):
1244 dupdir = os.path.join('duplicates', repodir)
1245 if not os.path.isdir(dupdir):
1246 os.makedirs(dupdir, exist_ok=True)
1247 dupfile = os.path.join('duplicates', std_long_name)
1248 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1249 os.rename(apkfile, dupfile)
1250 return True, None, False
1252 os.rename(apkfile, std_long_name)
1253 apkfile = std_long_name
1255 os.rename(apkfile, std_short_name)
1256 apkfile = std_short_name
1257 apkfilename = apkfile[len(repodir) + 1:]
1259 apk['apkName'] = apkfilename
1260 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1261 if os.path.exists(os.path.join(repodir, srcfilename)):
1262 apk['srcname'] = srcfilename
1264 # verify the jar signature is correct, allow deprecated
1265 # algorithms only if the APK is in the archive.
1267 if not common.verify_apk_signature(apkfile):
1268 if repodir == 'archive' or allow_disabled_algorithms:
1269 if common.verify_old_apk_signature(apkfile):
1270 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1278 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1279 move_apk_between_sections(repodir, 'archive', apk)
1281 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1282 return True, None, False
1284 apkzip = zipfile.ZipFile(apkfile, 'r')
1286 # if an APK has files newer than the system time, suggest updating
1287 # the system clock. This is useful for offline systems, used for
1288 # signing, which do not have another source of clock sync info. It
1289 # has to be more than 24 hours newer because ZIP/APK files do not
1290 # store timezone info
1291 manifest = apkzip.getinfo('AndroidManifest.xml')
1292 if manifest.date_time[1] == 0: # month can't be zero
1293 logging.debug('AndroidManifest.xml has no date')
1295 dt_obj = datetime(*manifest.date_time)
1296 checkdt = dt_obj - timedelta(1)
1297 if datetime.today() < checkdt:
1298 logging.warning('System clock is older than manifest in: '
1300 + '\nSet clock to that time using:\n'
1301 + 'sudo date -s "' + str(dt_obj) + '"')
1303 # extract icons from APK zip file
1304 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1306 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1308 apkzip.close() # ensure that APK zip file gets closed
1310 # resize existing icons for densities missing in the APK
1311 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1313 if use_date_from_apk and manifest.date_time[1] != 0:
1314 default_date_param = datetime(*manifest.date_time)
1316 default_date_param = None
1318 # Record in known apks, getting the added date at the same time..
1319 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1320 default_date=default_date_param)
1322 apk['added'] = added
1324 apkcache[apkfilename] = apk
1327 return False, apk, cachechanged
1330 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1331 """Processes the apks in the given repo directory.
1333 This also extracts the icons.
1335 :param apkcache: current apk cache information
1336 :param repodir: repo directory to scan
1337 :param knownapks: known apks info
1338 :param use_date_from_apk: use date from APK (instead of current date)
1339 for newly added APKs
1340 :returns: (apks, cachechanged) where apks is a list of apk information,
1341 and cachechanged is True if the apkcache got changed.
1344 cachechanged = False
1346 for icon_dir in get_all_icon_dirs(repodir):
1347 if os.path.exists(icon_dir):
1349 shutil.rmtree(icon_dir)
1350 os.makedirs(icon_dir)
1352 os.makedirs(icon_dir)
1355 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1356 apkfilename = apkfile[len(repodir) + 1:]
1357 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1358 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1359 use_date_from_apk, ada, True)
1363 cachechanged = cachechanged or cachethis
1365 return apks, cachechanged
1368 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1370 Extracts icons from the given APK zip in various densities,
1371 saves them into given repo directory
1372 and stores their names in the APK metadata dictionary.
1374 :param icon_filename: A string representing the icon's file name
1375 :param apk: A populated dictionary containing APK metadata.
1376 Needs to have 'icons_src' key
1377 :param apkzip: An opened zipfile.ZipFile of the APK file
1378 :param repo_dir: The directory of the APK's repository
1379 :return: A list of icon densities that are missing
1381 empty_densities = []
1382 for density in screen_densities:
1383 if density not in apk['icons_src']:
1384 empty_densities.append(density)
1386 icon_src = apk['icons_src'][density]
1387 icon_dir = get_icon_dir(repo_dir, density)
1388 icon_dest = os.path.join(icon_dir, icon_filename)
1390 # Extract the icon files per density
1391 if icon_src.endswith('.xml'):
1392 png = os.path.basename(icon_src)[:-4] + '.png'
1393 for f in apkzip.namelist():
1395 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1396 if m and screen_resolutions[m.group(2)] == density:
1398 if icon_src.endswith('.xml'):
1399 empty_densities.append(density)
1402 with open(icon_dest, 'wb') as f:
1403 f.write(get_icon_bytes(apkzip, icon_src))
1404 apk['icons'][density] = icon_filename
1405 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1406 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1407 del apk['icons_src'][density]
1408 empty_densities.append(density)
1410 if '-1' in apk['icons_src']:
1411 icon_src = apk['icons_src']['-1']
1412 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1413 with open(icon_path, 'wb') as f:
1414 f.write(get_icon_bytes(apkzip, icon_src))
1416 im = Image.open(icon_path)
1417 dpi = px_to_dpi(im.size[0])
1418 for density in screen_densities:
1419 if density in apk['icons']:
1421 if density == screen_densities[-1] or dpi >= int(density):
1422 apk['icons'][density] = icon_filename
1423 shutil.move(icon_path,
1424 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1425 empty_densities.remove(density)
1427 except Exception as e:
1428 logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1431 apk['icon'] = icon_filename
1433 return empty_densities
1436 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1438 Resize existing icons for densities missing in the APK to ensure all densities are available
1440 :param empty_densities: A list of icon densities that are missing
1441 :param icon_filename: A string representing the icon's file name
1442 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1443 :param repo_dir: The directory of the APK's repository
1445 # First try resizing down to not lose quality
1447 for density in screen_densities:
1448 if density not in empty_densities:
1449 last_density = density
1451 if last_density is None:
1453 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1455 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1456 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1459 fp = open(last_icon_path, 'rb')
1462 size = dpi_to_px(density)
1464 im.thumbnail((size, size), Image.ANTIALIAS)
1465 im.save(icon_path, "PNG")
1466 empty_densities.remove(density)
1467 except Exception as e:
1468 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1473 # Then just copy from the highest resolution available
1475 for density in reversed(screen_densities):
1476 if density not in empty_densities:
1477 last_density = density
1480 if last_density is None:
1484 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1485 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1487 empty_densities.remove(density)
1489 for density in screen_densities:
1490 icon_dir = get_icon_dir(repo_dir, density)
1491 icon_dest = os.path.join(icon_dir, icon_filename)
1492 resize_icon(icon_dest, density)
1494 # Copy from icons-mdpi to icons since mdpi is the baseline density
1495 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1496 if os.path.isfile(baseline):
1497 apk['icons']['0'] = icon_filename
1498 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1501 def apply_info_from_latest_apk(apps, apks):
1503 Some information from the apks needs to be applied up to the application level.
1504 When doing this, we use the info from the most recent version's apk.
1505 We deal with figuring out when the app was added and last updated at the same time.
1507 for appid, app in apps.items():
1508 bestver = UNSET_VERSION_CODE
1510 if apk['packageName'] == appid:
1511 if apk['versionCode'] > bestver:
1512 bestver = apk['versionCode']
1516 if not app.added or apk['added'] < app.added:
1517 app.added = apk['added']
1518 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1519 app.lastUpdated = apk['added']
1522 logging.debug("Don't know when " + appid + " was added")
1523 if not app.lastUpdated:
1524 logging.debug("Don't know when " + appid + " was last updated")
1526 if bestver == UNSET_VERSION_CODE:
1528 if app.Name is None:
1529 app.Name = app.AutoName or appid
1531 logging.debug("Application " + appid + " has no packages")
1533 if app.Name is None:
1534 app.Name = bestapk['name']
1535 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1536 if app.CurrentVersionCode is None:
1537 app.CurrentVersionCode = str(bestver)
1540 def make_categories_txt(repodir, categories):
1541 '''Write a category list in the repo to allow quick access'''
1543 for cat in sorted(categories):
1544 catdata += cat + '\n'
1545 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1549 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1551 def filter_apk_list_sorted(apk_list):
1553 for apk in apk_list:
1554 if apk['packageName'] == appid:
1557 # Sort the apk list by version code. First is highest/newest.
1558 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1560 for appid, app in apps.items():
1562 if app.ArchivePolicy:
1563 keepversions = int(app.ArchivePolicy[:-9])
1565 keepversions = defaultkeepversions
1567 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1568 .format(appid, len(apks), keepversions, len(archapks)))
1570 current_app_apks = filter_apk_list_sorted(apks)
1571 if len(current_app_apks) > keepversions:
1572 # Move back the ones we don't want.
1573 for apk in current_app_apks[keepversions:]:
1574 move_apk_between_sections(repodir, archivedir, apk)
1575 archapks.append(apk)
1578 current_app_archapks = filter_apk_list_sorted(archapks)
1579 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1581 # Move forward the ones we want again, except DisableAlgorithm
1582 for apk in current_app_archapks:
1583 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1584 move_apk_between_sections(archivedir, repodir, apk)
1585 archapks.remove(apk)
1588 if kept == keepversions:
1592 def move_apk_between_sections(from_dir, to_dir, apk):
1593 """move an APK from repo to archive or vice versa"""
1595 def _move_file(from_dir, to_dir, filename, ignore_missing):
1596 from_path = os.path.join(from_dir, filename)
1597 if ignore_missing and not os.path.exists(from_path):
1599 to_path = os.path.join(to_dir, filename)
1600 if not os.path.exists(to_dir):
1602 shutil.move(from_path, to_path)
1604 if from_dir == to_dir:
1607 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1608 _move_file(from_dir, to_dir, apk['apkName'], False)
1609 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1610 for density in all_screen_densities:
1611 from_icon_dir = get_icon_dir(from_dir, density)
1612 to_icon_dir = get_icon_dir(to_dir, density)
1613 if density not in apk['icons']:
1615 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1616 if 'srcname' in apk:
1617 _move_file(from_dir, to_dir, apk['srcname'], False)
1620 def add_apks_to_per_app_repos(repodir, apks):
1621 apks_per_app = dict()
1623 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1624 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1625 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1626 apks_per_app[apk['packageName']] = apk
1628 if not os.path.exists(apk['per_app_icons']):
1629 logging.info('Adding new repo for only ' + apk['packageName'])
1630 os.makedirs(apk['per_app_icons'])
1632 apkpath = os.path.join(repodir, apk['apkName'])
1633 shutil.copy(apkpath, apk['per_app_repo'])
1634 apksigpath = apkpath + '.sig'
1635 if os.path.exists(apksigpath):
1636 shutil.copy(apksigpath, apk['per_app_repo'])
1637 apkascpath = apkpath + '.asc'
1638 if os.path.exists(apkascpath):
1639 shutil.copy(apkascpath, apk['per_app_repo'])
1642 def create_metadata_from_template(apk):
1643 '''create a new metadata file using internal or external template
1645 Generate warnings for apk's with no metadata (or create skeleton
1646 metadata files, if requested on the command line). Though the
1647 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1648 since those impose things on the metadata file made from the
1649 template: field sort order, empty field value, formatting, etc.
1653 if os.path.exists('template.yml'):
1654 with open('template.yml') as f:
1656 if 'name' in apk and apk['name'] != '':
1657 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1658 r'\1 ' + apk['name'],
1660 flags=re.IGNORECASE | re.MULTILINE)
1662 logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1663 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1664 r'\1 ' + apk['packageName'],
1666 flags=re.IGNORECASE | re.MULTILINE)
1667 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1671 app['Categories'] = [os.path.basename(os.getcwd())]
1672 # include some blanks as part of the template
1673 app['AuthorName'] = ''
1676 app['IssueTracker'] = ''
1677 app['SourceCode'] = ''
1678 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1679 if 'name' in apk and apk['name'] != '':
1680 app['Name'] = apk['name']
1682 logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1683 app['Name'] = apk['packageName']
1684 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1685 yaml.dump(app, f, default_flow_style=False)
1686 logging.info("Generated skeleton metadata for " + apk['packageName'])
1695 global config, options
1697 # Parse command line...
1698 parser = ArgumentParser()
1699 common.setup_global_opts(parser)
1700 parser.add_argument("--create-key", action="store_true", default=False,
1701 help=_("Create a repo signing key in a keystore"))
1702 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1703 help=_("Create skeleton metadata files that are missing"))
1704 parser.add_argument("--delete-unknown", action="store_true", default=False,
1705 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1706 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1707 help=_("Report on build data status"))
1708 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1709 help=_("Interactively ask about things that need updating."))
1710 parser.add_argument("-I", "--icons", action="store_true", default=False,
1711 help=_("Resize all the icons exceeding the max pixel size and exit"))
1712 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1713 help=_("Specify editor to use in interactive mode. Default ") +
1714 "is /etc/alternatives/editor")
1715 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1716 help=_("Update the wiki"))
1717 parser.add_argument("--pretty", action="store_true", default=False,
1718 help=_("Produce human-readable index.xml"))
1719 parser.add_argument("--clean", action="store_true", default=False,
1720 help=_("Clean update - don't uses caches, reprocess all apks"))
1721 parser.add_argument("--nosign", action="store_true", default=False,
1722 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1723 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1724 help=_("Use date from apk instead of current time for newly added apks"))
1725 parser.add_argument("--rename-apks", action="store_true", default=False,
1726 help=_("Rename APK files that do not match package.name_123.apk"))
1727 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1728 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1729 metadata.add_metadata_arguments(parser)
1730 options = parser.parse_args()
1731 metadata.warnings_action = options.W
1733 config = common.read_config(options)
1735 if not ('jarsigner' in config and 'keytool' in config):
1736 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1739 if config['archive_older'] != 0:
1740 repodirs.append('archive')
1741 if not os.path.exists('archive'):
1745 resize_all_icons(repodirs)
1748 if options.rename_apks:
1749 options.clean = True
1751 # check that icons exist now, rather than fail at the end of `fdroid update`
1752 for k in ['repo_icon', 'archive_icon']:
1754 if not os.path.exists(config[k]):
1755 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1758 # if the user asks to create a keystore, do it now, reusing whatever it can
1759 if options.create_key:
1760 if os.path.exists(config['keystore']):
1761 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1762 logging.critical("\t'" + config['keystore'] + "'")
1765 if 'repo_keyalias' not in config:
1766 config['repo_keyalias'] = socket.getfqdn()
1767 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1768 if 'keydname' not in config:
1769 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1770 common.write_to_config(config, 'keydname', config['keydname'])
1771 if 'keystore' not in config:
1772 config['keystore'] = common.default_config['keystore']
1773 common.write_to_config(config, 'keystore', config['keystore'])
1775 password = common.genpassword()
1776 if 'keystorepass' not in config:
1777 config['keystorepass'] = password
1778 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1779 if 'keypass' not in config:
1780 config['keypass'] = password
1781 common.write_to_config(config, 'keypass', config['keypass'])
1782 common.genkeystore(config)
1785 apps = metadata.read_metadata()
1787 # Generate a list of categories...
1789 for app in apps.values():
1790 categories.update(app.Categories)
1792 # Read known apks data (will be updated and written back when we've finished)
1793 knownapks = common.KnownApks()
1796 apkcache = get_cache()
1798 # Delete builds for disabled apps
1799 delete_disabled_builds(apps, apkcache, repodirs)
1801 # Scan all apks in the main repo
1802 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1804 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1805 options.use_date_from_apk)
1806 cachechanged = cachechanged or fcachechanged
1809 if apk['packageName'] not in apps:
1810 if options.create_metadata:
1811 create_metadata_from_template(apk)
1812 apps = metadata.read_metadata()
1814 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1815 if options.delete_unknown:
1816 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1817 rmf = os.path.join(repodirs[0], apk['apkName'])
1818 if not os.path.exists(rmf):
1819 logging.error("Could not find {0} to remove it".format(rmf))
1823 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1825 copy_triple_t_store_metadata(apps)
1826 insert_obbs(repodirs[0], apps, apks)
1827 insert_localized_app_metadata(apps)
1828 translate_per_build_anti_features(apps, apks)
1830 # Scan the archive repo for apks as well
1831 if len(repodirs) > 1:
1832 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1838 # Apply information from latest apks to the application and update dates
1839 apply_info_from_latest_apk(apps, apks + archapks)
1841 # Sort the app list by name, then the web site doesn't have to by default.
1842 # (we had to wait until we'd scanned the apks to do this, because mostly the
1843 # name comes from there!)
1844 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1846 # APKs are placed into multiple repos based on the app package, providing
1847 # per-app subscription feeds for nightly builds and things like it
1848 if config['per_app_repos']:
1849 add_apks_to_per_app_repos(repodirs[0], apks)
1850 for appid, app in apps.items():
1851 repodir = os.path.join(appid, 'fdroid', 'repo')
1853 appdict[appid] = app
1854 if os.path.isdir(repodir):
1855 index.make(appdict, [appid], apks, repodir, False)
1857 logging.info('Skipping index generation for ' + appid)
1860 if len(repodirs) > 1:
1861 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1863 # Make the index for the main repo...
1864 index.make(apps, sortedids, apks, repodirs[0], False)
1865 make_categories_txt(repodirs[0], categories)
1867 # If there's an archive repo, make the index for it. We already scanned it
1869 if len(repodirs) > 1:
1870 index.make(apps, sortedids, archapks, repodirs[1], True)
1872 git_remote = config.get('binary_transparency_remote')
1873 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1875 btlog.make_binary_transparency_log(repodirs)
1877 if config['update_stats']:
1878 # Update known apks info...
1879 knownapks.writeifchanged()
1881 # Generate latest apps data for widget
1882 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1884 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1886 appid = line.rstrip()
1887 data += appid + "\t"
1889 data += app.Name + "\t"
1890 if app.icon is not None:
1891 data += app.icon + "\t"
1892 data += app.License + "\n"
1893 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1897 write_cache(apkcache)
1899 # Update the wiki...
1901 update_wiki(apps, sortedids, apks + archapks)
1903 logging.info(_("Finished"))
1906 if __name__ == "__main__":