3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
42 from . import metadata
43 from .common import SdkToolsPopen
44 from .exception import BuildException, FDroidException
48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
49 UNSET_VERSION_CODE = -0x100000000
51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
58 APK_PERMISSION_PAT = \
59 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
62 screen_densities = ['640', '480', '320', '240', '160', '120']
63 screen_resolutions = {
75 all_screen_densities = ['0'] + screen_densities
77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
83 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
86 def dpi_to_px(density):
87 return (int(density) * 48) / 160
91 return (int(px) * 160) / 48
94 def get_icon_dir(repodir, density):
96 return os.path.join(repodir, "icons")
97 return os.path.join(repodir, "icons-%s" % density)
100 def get_icon_dirs(repodir):
101 for density in screen_densities:
102 yield get_icon_dir(repodir, density)
105 def get_all_icon_dirs(repodir):
106 for density in all_screen_densities:
107 yield get_icon_dir(repodir, density)
110 def update_wiki(apps, sortedids, apks):
113 :param apps: fully populated list of all applications
114 :param apks: all apks, except...
116 logging.info("Updating wiki")
118 wikiredircat = 'App Redirects'
120 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
121 path=config['wiki_path'])
122 site.login(config['wiki_user'], config['wiki_password'])
124 generated_redirects = {}
126 for appid in sortedids:
127 app = metadata.App(apps[appid])
131 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
133 for af in app.AntiFeatures:
134 wikidata += '{{AntiFeature|' + af + '}}\n'
139 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
142 app.added.strftime('%Y-%m-%d') if app.added else '',
143 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
158 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
160 wikidata += app.Summary
161 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
163 wikidata += "=Description=\n"
164 wikidata += metadata.description_wiki(app.Description) + "\n"
166 wikidata += "=Maintainer Notes=\n"
167 if app.MaintainerNotes:
168 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
169 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
171 # Get a list of all packages for this application...
173 gotcurrentver = False
177 if apk['packageName'] == appid:
178 if str(apk['versionCode']) == app.CurrentVersionCode:
181 # Include ones we can't build, as a special case...
182 for build in app.builds:
184 if build.versionCode == app.CurrentVersionCode:
186 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
187 apklist.append({'versionCode': int(build.versionCode),
188 'versionName': build.versionName,
189 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
194 if apk['versionCode'] == int(build.versionCode):
199 apklist.append({'versionCode': int(build.versionCode),
200 'versionName': build.versionName,
201 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
203 if app.CurrentVersionCode == '0':
205 # Sort with most recent first...
206 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
208 wikidata += "=Versions=\n"
209 if len(apklist) == 0:
210 wikidata += "We currently have no versions of this app available."
211 elif not gotcurrentver:
212 wikidata += "We don't have the current version of this app."
214 wikidata += "We have the current version of this app."
215 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
216 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
217 if len(app.NoSourceSince) > 0:
218 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
219 if len(app.CurrentVersion) > 0:
220 wikidata += "The current (recommended) version is " + app.CurrentVersion
221 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
224 wikidata += "==" + apk['versionName'] + "==\n"
226 if 'buildproblem' in apk:
227 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
230 wikidata += "This version is built and signed by "
232 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
234 wikidata += "the original developer.\n\n"
235 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
237 wikidata += '\n[[Category:' + wikicat + ']]\n'
238 if len(app.NoSourceSince) > 0:
239 wikidata += '\n[[Category:Apps missing source code]]\n'
240 if validapks == 0 and not app.Disabled:
241 wikidata += '\n[[Category:Apps with no packages]]\n'
242 if cantupdate and not app.Disabled:
243 wikidata += "\n[[Category:Apps we cannot update]]\n"
244 if buildfails and not app.Disabled:
245 wikidata += "\n[[Category:Apps with failing builds]]\n"
246 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
247 wikidata += '\n[[Category:Apps to Update]]\n'
249 wikidata += '\n[[Category:Apps that are disabled]]\n'
250 if app.UpdateCheckMode == 'None' and not app.Disabled:
251 wikidata += '\n[[Category:Apps with no update check]]\n'
252 for appcat in app.Categories:
253 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
255 # We can't have underscores in the page name, even if they're in
256 # the package ID, because MediaWiki messes with them...
257 pagename = appid.replace('_', ' ')
259 # Drop a trailing newline, because mediawiki is going to drop it anyway
260 # and it we don't we'll think the page has changed when it hasn't...
261 if wikidata.endswith('\n'):
262 wikidata = wikidata[:-1]
264 generated_pages[pagename] = wikidata
266 # Make a redirect from the name to the ID too, unless there's
267 # already an existing page with the name and it isn't a redirect.
269 apppagename = app.Name.replace('_', ' ')
270 apppagename = apppagename.replace('{', '')
271 apppagename = apppagename.replace('}', ' ')
272 apppagename = apppagename.replace(':', ' ')
273 apppagename = apppagename.replace('[', ' ')
274 apppagename = apppagename.replace(']', ' ')
275 # Drop double spaces caused mostly by replacing ':' above
276 apppagename = apppagename.replace(' ', ' ')
277 for expagename in site.allpages(prefix=apppagename,
278 filterredir='nonredirects',
280 if expagename == apppagename:
282 # Another reason not to make the redirect page is if the app name
283 # is the same as it's ID, because that will overwrite the real page
284 # with an redirect to itself! (Although it seems like an odd
285 # scenario this happens a lot, e.g. where there is metadata but no
286 # builds or binaries to extract a name from.
287 if apppagename == pagename:
290 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
292 for tcat, genp in [(wikicat, generated_pages),
293 (wikiredircat, generated_redirects)]:
294 catpages = site.Pages['Category:' + tcat]
296 for page in catpages:
297 existingpages.append(page.name)
298 if page.name in genp:
299 pagetxt = page.edit()
300 if pagetxt != genp[page.name]:
301 logging.debug("Updating modified page " + page.name)
302 page.save(genp[page.name], summary='Auto-updated')
304 logging.debug("Page " + page.name + " is unchanged")
306 logging.warn("Deleting page " + page.name)
307 page.delete('No longer published')
308 for pagename, text in genp.items():
309 logging.debug("Checking " + pagename)
310 if pagename not in existingpages:
311 logging.debug("Creating page " + pagename)
313 newpage = site.Pages[pagename]
314 newpage.save(text, summary='Auto-created')
315 except Exception as e:
316 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
318 # Purge server cache to ensure counts are up to date
319 site.pages['Repository Maintenance'].purge()
322 def delete_disabled_builds(apps, apkcache, repodirs):
323 """Delete disabled build outputs.
325 :param apps: list of all applications, as per metadata.read_metadata
326 :param apkcache: current apk cache information
327 :param repodirs: the repo directories to process
329 for appid, app in apps.items():
330 for build in app['builds']:
331 if not build.disable:
333 apkfilename = common.get_release_filename(app, build)
334 iconfilename = "%s.%s.png" % (
337 for repodir in repodirs:
339 os.path.join(repodir, apkfilename),
340 os.path.join(repodir, apkfilename + '.asc'),
341 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
343 for density in all_screen_densities:
344 repo_dir = get_icon_dir(repodir, density)
345 files.append(os.path.join(repo_dir, iconfilename))
348 if os.path.exists(f):
349 logging.info("Deleting disabled build output " + f)
351 if apkfilename in apkcache:
352 del apkcache[apkfilename]
355 def resize_icon(iconpath, density):
357 if not os.path.isfile(iconpath):
362 fp = open(iconpath, 'rb')
364 size = dpi_to_px(density)
366 if any(length > size for length in im.size):
368 im.thumbnail((size, size), Image.ANTIALIAS)
369 logging.debug("%s was too large at %s - new size is %s" % (
370 iconpath, oldsize, im.size))
371 im.save(iconpath, "PNG")
373 except Exception as e:
374 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
381 def resize_all_icons(repodirs):
382 """Resize all icons that exceed the max size
384 :param repodirs: the repo directories to process
386 for repodir in repodirs:
387 for density in screen_densities:
388 icon_dir = get_icon_dir(repodir, density)
389 icon_glob = os.path.join(icon_dir, '*.png')
390 for iconpath in glob.glob(icon_glob):
391 resize_icon(iconpath, density)
395 """ Get the signing certificate of an apk. To get the same md5 has that
396 Android gets, we encode the .RSA certificate in a specific format and pass
397 it hex-encoded to the md5 digest algorithm.
399 :param apkpath: path to the apk
400 :returns: A string containing the md5 of the signature of the apk or None
401 if an error occurred.
404 with zipfile.ZipFile(apkpath, 'r') as apk:
405 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
408 logging.error("Found no signing certificates on %s" % apkpath)
411 logging.error("Found multiple signing certificates on %s" % apkpath)
414 cert = apk.read(certs[0])
416 cert_encoded = common.get_certificate(cert)
418 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
421 def get_cache_file():
422 return os.path.join('tmp', 'apkcache')
426 """Get the cached dict of the APK index
428 Gather information about all the apk files in the repo directory,
429 using cached data if possible. Some of the index operations take a
430 long time, like calculating the SHA-256 and verifying the APK
433 The cache is invalidated if the metadata version is different, or
434 the 'allow_disabled_algorithms' config/option is different. In
435 those cases, there is no easy way to know what has changed from
436 the cache, so just rerun the whole thing.
441 apkcachefile = get_cache_file()
442 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
443 if not options.clean and os.path.exists(apkcachefile):
444 with open(apkcachefile, 'rb') as cf:
445 apkcache = pickle.load(cf, encoding='utf-8')
446 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
447 or apkcache.get('allow_disabled_algorithms') != ada:
452 apkcache["METADATA_VERSION"] = METADATA_VERSION
453 apkcache['allow_disabled_algorithms'] = ada
458 def write_cache(apkcache):
459 apkcachefile = get_cache_file()
460 cache_path = os.path.dirname(apkcachefile)
461 if not os.path.exists(cache_path):
462 os.makedirs(cache_path)
463 with open(apkcachefile, 'wb') as cf:
464 pickle.dump(apkcache, cf)
467 def get_icon_bytes(apkzip, iconsrc):
468 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
470 return apkzip.read(iconsrc)
472 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
475 def sha256sum(filename):
476 '''Calculate the sha256 of the given file'''
477 sha = hashlib.sha256()
478 with open(filename, 'rb') as f:
484 return sha.hexdigest()
487 def has_known_vulnerability(filename):
488 """checks for known vulnerabilities in the APK
490 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
491 version. Google also enforces this:
492 https://support.google.com/faqs/answer/6376725?hl=en
494 Checks whether there are more than one classes.dex or AndroidManifest.xml
495 files, which is invalid and an essential part of the "Master Key" attack.
497 http://www.saurik.com/id/17
500 # statically load this pattern
501 if not hasattr(has_known_vulnerability, "pattern"):
502 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
505 with zipfile.ZipFile(filename) as zf:
506 for name in zf.namelist():
507 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
510 chunk = lib.read(4096)
513 m = has_known_vulnerability.pattern.search(chunk)
515 version = m.group(1).decode('ascii')
516 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
517 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
518 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
519 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
521 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
524 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
525 if name in files_in_apk:
527 files_in_apk.add(name)
532 def insert_obbs(repodir, apps, apks):
533 """Scans the .obb files in a given repo directory and adds them to the
534 relevant APK instances. OBB files have versionCodes like APK
535 files, and they are loosely associated. If there is an OBB file
536 present, then any APK with the same or higher versionCode will use
537 that OBB file. There are two OBB types: main and patch, each APK
538 can only have only have one of each.
540 https://developer.android.com/google/play/expansion-files.html
542 :param repodir: repo directory to scan
543 :param apps: list of current, valid apps
544 :param apks: current information on all APKs
548 def obbWarnDelete(f, msg):
549 logging.warning(msg + f)
550 if options.delete_unknown:
551 logging.error("Deleting unknown file: " + f)
555 java_Integer_MIN_VALUE = -pow(2, 31)
556 currentPackageNames = apps.keys()
557 for f in glob.glob(os.path.join(repodir, '*.obb')):
558 obbfile = os.path.basename(f)
559 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
560 chunks = obbfile.split('.')
561 if chunks[0] != 'main' and chunks[0] != 'patch':
562 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
564 if not re.match(r'^-?[0-9]+$', chunks[1]):
565 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
567 versionCode = int(chunks[1])
568 packagename = ".".join(chunks[2:-1])
570 highestVersionCode = java_Integer_MIN_VALUE
571 if packagename not in currentPackageNames:
572 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
575 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
576 highestVersionCode = apk['versionCode']
577 if versionCode > highestVersionCode:
578 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
579 + ') than any APK: ')
581 obbsha256 = sha256sum(f)
582 obbs.append((packagename, versionCode, obbfile, obbsha256))
585 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
586 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
587 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
588 apk['obbMainFile'] = obbfile
589 apk['obbMainFileSha256'] = obbsha256
590 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
591 apk['obbPatchFile'] = obbfile
592 apk['obbPatchFileSha256'] = obbsha256
593 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
597 def translate_per_build_anti_features(apps, apks):
598 """Grab the anti-features list from the build metadata
600 For most Anti-Features, they are really most applicable per-APK,
601 not for an app. An app can fix a vulnerability, add/remove
602 tracking, etc. This reads the 'antifeatures' list from the Build
603 entries in the fdroiddata metadata file, then transforms it into
604 the 'antiFeatures' list of unique items for the index.
606 The field key is all lower case in the metadata file to match the
607 rest of the Build fields. It is 'antiFeatures' camel case in the
608 implementation, index, and fdroidclient since it is translated
609 from the build 'antifeatures' field, not directly included.
613 antiFeatures = dict()
614 for packageName, app in apps.items():
616 for build in app['builds']:
617 afl = build.get('antifeatures')
619 d[int(build.versionCode)] = afl
621 antiFeatures[packageName] = d
624 d = antiFeatures.get(apk['packageName'])
626 afl = d.get(apk['versionCode'])
628 apk['antiFeatures'].update(afl)
631 def _get_localized_dict(app, locale):
632 '''get the dict to add localized store metadata to'''
633 if 'localized' not in app:
634 app['localized'] = collections.OrderedDict()
635 if locale not in app['localized']:
636 app['localized'][locale] = collections.OrderedDict()
637 return app['localized'][locale]
640 def _set_localized_text_entry(app, locale, key, f):
641 limit = config['char_limits'][key]
642 localized = _get_localized_dict(app, locale)
644 text = fp.read()[:limit]
646 localized[key] = text
649 def _set_author_entry(app, key, f):
650 limit = config['char_limits']['author']
652 text = fp.read()[:limit]
657 def copy_triple_t_store_metadata(apps):
658 """Include store metadata from the app's source repo
660 The Triple-T Gradle Play Publisher is a plugin that has a standard
661 file layout for all of the metadata and graphics that the Google
662 Play Store accepts. Since F-Droid has the git repo, it can just
663 pluck those files directly. This method reads any text files into
664 the app dict, then copies any graphics into the fdroid repo
667 This needs to be run before insert_localized_app_metadata() so that
668 the graphics files that are copied into the fdroid repo get
671 https://github.com/Triple-T/gradle-play-publisher#upload-images
672 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
676 if not os.path.isdir('build'):
677 return # nothing to do
679 for packageName, app in apps.items():
680 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
681 logging.debug('Triple-T Gradle Play Publisher: ' + d)
682 for root, dirs, files in os.walk(d):
683 segments = root.split('/')
684 locale = segments[-2]
686 if f == 'fulldescription':
687 _set_localized_text_entry(app, locale, 'description',
688 os.path.join(root, f))
690 elif f == 'shortdescription':
691 _set_localized_text_entry(app, locale, 'summary',
692 os.path.join(root, f))
695 _set_localized_text_entry(app, locale, 'name',
696 os.path.join(root, f))
699 _set_localized_text_entry(app, locale, 'video',
700 os.path.join(root, f))
702 elif f == 'whatsnew':
703 _set_localized_text_entry(app, segments[-1], 'whatsNew',
704 os.path.join(root, f))
706 elif f == 'contactEmail':
707 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
709 elif f == 'contactPhone':
710 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
712 elif f == 'contactWebsite':
713 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
716 base, extension = common.get_extension(f)
717 dirname = os.path.basename(root)
718 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
719 if segments[-2] == 'listing':
720 locale = segments[-3]
722 locale = segments[-2]
723 destdir = os.path.join('repo', packageName, locale)
724 os.makedirs(destdir, mode=0o755, exist_ok=True)
725 sourcefile = os.path.join(root, f)
726 destfile = os.path.join(destdir, dirname + '.' + extension)
727 logging.debug('copying ' + sourcefile + ' ' + destfile)
728 shutil.copy(sourcefile, destfile)
731 def insert_localized_app_metadata(apps):
732 """scans standard locations for graphics and localized text
734 Scans for localized description files, store graphics, and
735 screenshot PNG files in statically defined screenshots directory
736 and adds them to the app metadata. The screenshots and graphic
737 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
738 and must be in the following layout:
739 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
741 repo/packageName/locale/featureGraphic.png
742 repo/packageName/locale/phoneScreenshots/1.png
743 repo/packageName/locale/phoneScreenshots/2.png
745 The changelog files must be text files named with the versionCode
746 ending with ".txt" and must be in the following layout:
747 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
749 repo/packageName/locale/changelogs/12345.txt
751 This will scan the each app's source repo then the metadata/ dir
752 for these standard locations of changelog files. If it finds
753 them, they will be added to the dict of all packages, with the
754 versions in the metadata/ folder taking precendence over the what
755 is in the app's source repo.
757 Where "packageName" is the app's packageName and "locale" is the locale
758 of the graphics, e.g. what language they are in, using the IETF RFC5646
759 format (en-US, fr-CA, es-MX, etc).
761 This will also scan the app's git for a fastlane folder, and the
762 metadata/ folder and the apps' source repos for standard locations
763 of graphic and screenshot files. If it finds them, it will copy
764 them into the repo. The fastlane files follow this pattern:
765 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
769 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
770 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
771 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
773 for srcd in sorted(sourcedirs):
774 if not os.path.isdir(srcd):
776 for root, dirs, files in os.walk(srcd):
777 segments = root.split('/')
778 packageName = segments[1]
779 if packageName not in apps:
780 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
782 locale = segments[-1]
783 destdir = os.path.join('repo', packageName, locale)
785 if f in ('description.txt', 'full_description.txt'):
786 _set_localized_text_entry(apps[packageName], locale, 'description',
787 os.path.join(root, f))
789 elif f in ('summary.txt', 'short_description.txt'):
790 _set_localized_text_entry(apps[packageName], locale, 'summary',
791 os.path.join(root, f))
793 elif f in ('name.txt', 'title.txt'):
794 _set_localized_text_entry(apps[packageName], locale, 'name',
795 os.path.join(root, f))
797 elif f == 'video.txt':
798 _set_localized_text_entry(apps[packageName], locale, 'video',
799 os.path.join(root, f))
801 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
802 locale = segments[-2]
803 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
804 os.path.join(root, f))
807 base, extension = common.get_extension(f)
808 if locale == 'images':
809 locale = segments[-2]
810 destdir = os.path.join('repo', packageName, locale)
811 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
812 os.makedirs(destdir, mode=0o755, exist_ok=True)
813 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
814 shutil.copy(os.path.join(root, f), destdir)
816 if d in SCREENSHOT_DIRS:
817 for f in glob.glob(os.path.join(root, d, '*.*')):
818 _, extension = common.get_extension(f)
819 if extension in ALLOWED_EXTENSIONS:
820 screenshotdestdir = os.path.join(destdir, d)
821 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
822 logging.debug('copying ' + f + ' ' + screenshotdestdir)
823 shutil.copy(f, screenshotdestdir)
825 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
827 if not os.path.isdir(d):
829 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
830 if not os.path.isfile(f):
832 segments = f.split('/')
833 packageName = segments[1]
835 screenshotdir = segments[3]
836 filename = os.path.basename(f)
837 base, extension = common.get_extension(filename)
839 if packageName not in apps:
840 logging.warning('Found "%s" graphic without metadata for app "%s"!'
841 % (filename, packageName))
843 graphics = _get_localized_dict(apps[packageName], locale)
845 if extension not in ALLOWED_EXTENSIONS:
846 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
847 elif base in GRAPHIC_NAMES:
848 # there can only be zero or one of these per locale
849 graphics[base] = filename
850 elif screenshotdir in SCREENSHOT_DIRS:
851 # there can any number of these per locale
852 logging.debug('adding to ' + screenshotdir + ': ' + f)
853 if screenshotdir not in graphics:
854 graphics[screenshotdir] = []
855 graphics[screenshotdir].append(filename)
857 logging.warning('Unsupported graphics file found: ' + f)
860 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
861 """Scan a repo for all files with an extension except APK/OBB
863 :param apkcache: current cached info about all repo files
864 :param repodir: repo directory to scan
865 :param knownapks: list of all known files, as per metadata.read_metadata
866 :param use_date_from_file: use date from file (instead of current date)
867 for newly added files
872 repodir = repodir.encode('utf-8')
873 for name in os.listdir(repodir):
874 file_extension = common.get_file_extension(name)
875 if file_extension == 'apk' or file_extension == 'obb':
877 filename = os.path.join(repodir, name)
878 name_utf8 = name.decode('utf-8')
879 if filename.endswith(b'_src.tar.gz'):
880 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
882 if not common.is_repo_file(filename):
884 stat = os.stat(filename)
885 if stat.st_size == 0:
886 raise FDroidException(filename + ' is zero size!')
888 shasum = sha256sum(filename)
891 repo_file = apkcache[name]
892 # added time is cached as tuple but used here as datetime instance
893 if 'added' in repo_file:
894 a = repo_file['added']
895 if isinstance(a, datetime):
896 repo_file['added'] = a
898 repo_file['added'] = datetime(*a[:6])
899 if repo_file.get('hash') == shasum:
900 logging.debug("Reading " + name_utf8 + " from cache")
903 logging.debug("Ignoring stale cache data for " + name)
906 logging.debug("Processing " + name_utf8)
907 repo_file = collections.OrderedDict()
908 repo_file['name'] = os.path.splitext(name_utf8)[0]
909 # TODO rename apkname globally to something more generic
910 repo_file['apkName'] = name_utf8
911 repo_file['hash'] = shasum
912 repo_file['hashType'] = 'sha256'
913 repo_file['versionCode'] = 0
914 repo_file['versionName'] = shasum
915 # the static ID is the SHA256 unless it is set in the metadata
916 repo_file['packageName'] = shasum
918 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
920 repo_file['packageName'] = m.group(1)
921 repo_file['versionCode'] = int(m.group(2))
922 srcfilename = name + b'_src.tar.gz'
923 if os.path.exists(os.path.join(repodir, srcfilename)):
924 repo_file['srcname'] = srcfilename.decode('utf-8')
925 repo_file['size'] = stat.st_size
927 apkcache[name] = repo_file
930 if use_date_from_file:
931 timestamp = stat.st_ctime
932 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
934 default_date_param = None
936 # Record in knownapks, getting the added date at the same time..
937 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
938 default_date=default_date_param)
940 repo_file['added'] = added
942 repo_files.append(repo_file)
944 return repo_files, cachechanged
947 def scan_apk(apk_file):
949 Scans an APK file and returns dictionary with metadata of the APK.
951 Attention: This does *not* verify that the APK signature is correct.
953 :param apk_file: The (ideally absolute) path to the APK file
954 :raises BuildException
955 :return A dict containing APK metadata
958 'hash': sha256sum(apk_file),
959 'hashType': 'sha256',
960 'uses-permission': [],
961 'uses-permission-sdk-23': [],
965 'antiFeatures': set(),
968 if SdkToolsPopen(['aapt', 'version'], output=False):
969 scan_apk_aapt(apk, apk_file)
971 scan_apk_androguard(apk, apk_file)
974 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
975 apk['sig'] = getsig(apk_file)
977 raise BuildException("Failed to get apk signature")
979 # Get size of the APK
980 apk['size'] = os.path.getsize(apk_file)
982 if 'minSdkVersion' not in apk:
983 logging.warning("No SDK version information found in {0}".format(apk_file))
984 apk['minSdkVersion'] = 1
986 # Check for known vulnerabilities
987 if has_known_vulnerability(apk_file):
988 apk['antiFeatures'].add('KnownVuln')
993 def scan_apk_aapt(apk, apkfile):
994 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
995 if p.returncode != 0:
996 if options.delete_unknown:
997 if os.path.exists(apkfile):
998 logging.error("Failed to get apk information, deleting " + apkfile)
1001 logging.error("Could not find {0} to remove it".format(apkfile))
1003 logging.error("Failed to get apk information, skipping " + apkfile)
1004 raise BuildException("Invalid APK")
1005 for line in p.output.splitlines():
1006 if line.startswith("package:"):
1008 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1009 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1010 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1011 except Exception as e:
1012 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1013 elif line.startswith("application:"):
1014 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1015 # Keep path to non-dpi icon in case we need it
1016 match = re.match(APK_ICON_PAT_NODPI, line)
1018 apk['icons_src']['-1'] = match.group(1)
1019 elif line.startswith("launchable-activity:"):
1020 # Only use launchable-activity as fallback to application
1022 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1023 if '-1' not in apk['icons_src']:
1024 match = re.match(APK_ICON_PAT_NODPI, line)
1026 apk['icons_src']['-1'] = match.group(1)
1027 elif line.startswith("application-icon-"):
1028 match = re.match(APK_ICON_PAT, line)
1030 density = match.group(1)
1031 path = match.group(2)
1032 apk['icons_src'][density] = path
1033 elif line.startswith("sdkVersion:"):
1034 m = re.match(APK_SDK_VERSION_PAT, line)
1036 logging.error(line.replace('sdkVersion:', '')
1037 + ' is not a valid minSdkVersion!')
1039 apk['minSdkVersion'] = m.group(1)
1040 # if target not set, default to min
1041 if 'targetSdkVersion' not in apk:
1042 apk['targetSdkVersion'] = m.group(1)
1043 elif line.startswith("targetSdkVersion:"):
1044 m = re.match(APK_SDK_VERSION_PAT, line)
1046 logging.error(line.replace('targetSdkVersion:', '')
1047 + ' is not a valid targetSdkVersion!')
1049 apk['targetSdkVersion'] = m.group(1)
1050 elif line.startswith("maxSdkVersion:"):
1051 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1052 elif line.startswith("native-code:"):
1053 apk['nativecode'] = []
1054 for arch in line[13:].split(' '):
1055 apk['nativecode'].append(arch[1:-1])
1056 elif line.startswith('uses-permission:'):
1057 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1058 if perm_match['maxSdkVersion']:
1059 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1060 permission = UsesPermission(
1062 perm_match['maxSdkVersion']
1065 apk['uses-permission'].append(permission)
1066 elif line.startswith('uses-permission-sdk-23:'):
1067 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1068 if perm_match['maxSdkVersion']:
1069 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1070 permission_sdk_23 = UsesPermissionSdk23(
1072 perm_match['maxSdkVersion']
1075 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1077 elif line.startswith('uses-feature:'):
1078 feature = re.match(APK_FEATURE_PAT, line).group(1)
1079 # Filter out this, it's only added with the latest SDK tools and
1080 # causes problems for lots of apps.
1081 if feature != "android.hardware.screen.portrait" \
1082 and feature != "android.hardware.screen.landscape":
1083 if feature.startswith("android.feature."):
1084 feature = feature[16:]
1085 apk['features'].add(feature)
1088 def scan_apk_androguard(apk, apkfile):
1090 from androguard.core.bytecodes.apk import APK
1091 apkobject = APK(apkfile)
1092 if apkobject.is_valid_APK():
1093 arsc = apkobject.get_android_resources()
1095 if options.delete_unknown:
1096 if os.path.exists(apkfile):
1097 logging.error("Failed to get apk information, deleting " + apkfile)
1100 logging.error("Could not find {0} to remove it".format(apkfile))
1102 logging.error("Failed to get apk information, skipping " + apkfile)
1103 raise BuildException("Invaild APK")
1105 raise FDroidException("androguard library is not installed and aapt not present")
1106 except FileNotFoundError:
1107 logging.error("Could not open apk file for analysis")
1108 raise BuildException("Invalid APK")
1110 apk['packageName'] = apkobject.get_package()
1111 apk['versionCode'] = int(apkobject.get_androidversion_code())
1112 apk['versionName'] = apkobject.get_androidversion_name()
1113 if apk['versionName'][0] == "@":
1114 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1115 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1116 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1117 apk['name'] = apkobject.get_app_name()
1119 if apkobject.get_max_sdk_version() is not None:
1120 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1121 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1122 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1124 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1125 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1127 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1129 for file in apkobject.get_files():
1130 d_re = density_re.match(file)
1132 folder = d_re.group(1).split('-')
1134 resolution = folder[1]
1137 density = screen_resolutions[resolution]
1138 apk['icons_src'][density] = d_re.group(0)
1140 if apk['icons_src'].get('-1') is None:
1141 apk['icons_src']['-1'] = apk['icons_src']['160']
1143 arch_re = re.compile("^lib/(.*)/.*$")
1144 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1146 apk['nativecode'] = []
1147 apk['nativecode'].extend(sorted(list(arch)))
1149 xml = apkobject.get_android_manifest_xml()
1151 for item in xml.getElementsByTagName('uses-permission'):
1152 name = str(item.getAttribute("android:name"))
1153 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1154 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1155 permission = UsesPermission(
1159 apk['uses-permission'].append(permission)
1161 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1162 name = str(item.getAttribute("android:name"))
1163 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1164 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1165 permission_sdk_23 = UsesPermissionSdk23(
1169 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1171 for item in xml.getElementsByTagName('uses-feature'):
1172 feature = str(item.getAttribute("android:name"))
1173 if feature != "android.hardware.screen.portrait" \
1174 and feature != "android.hardware.screen.landscape":
1175 if feature.startswith("android.feature."):
1176 feature = feature[16:]
1177 apk['features'].append(feature)
1180 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1181 allow_disabled_algorithms=False, archive_bad_sig=False):
1182 """Processes the apk with the given filename in the given repo directory.
1184 This also extracts the icons.
1186 :param apkcache: current apk cache information
1187 :param apkfilename: the filename of the apk to scan
1188 :param repodir: repo directory to scan
1189 :param knownapks: known apks info
1190 :param use_date_from_apk: use date from APK (instead of current date)
1191 for newly added APKs
1192 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1193 disabled algorithms in the signature (e.g. MD5)
1194 :param archive_bad_sig: move APKs with a bad signature to the archive
1195 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1196 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1199 if ' ' in apkfilename:
1200 if options.rename_apks:
1201 newfilename = apkfilename.replace(' ', '_')
1202 os.rename(os.path.join(repodir, apkfilename),
1203 os.path.join(repodir, newfilename))
1204 apkfilename = newfilename
1206 logging.critical("Spaces in filenames are not allowed.")
1207 return True, None, False
1210 apkfile = os.path.join(repodir, apkfilename)
1212 cachechanged = False
1214 if apkfilename in apkcache:
1215 apk = apkcache[apkfilename]
1216 if apk.get('hash') == sha256sum(apkfile):
1217 logging.debug("Reading " + apkfilename + " from cache")
1220 logging.debug("Ignoring stale cache data for " + apkfilename)
1223 logging.debug("Processing " + apkfilename)
1226 apk = scan_apk(apkfile)
1227 except BuildException:
1228 logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1229 return True, None, False
1231 # Check for debuggable apks...
1232 if common.isApkAndDebuggable(apkfile):
1233 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1235 if options.rename_apks:
1236 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1237 std_short_name = os.path.join(repodir, n)
1238 if apkfile != std_short_name:
1239 if os.path.exists(std_short_name):
1240 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1241 if apkfile != std_long_name:
1242 if os.path.exists(std_long_name):
1243 dupdir = os.path.join('duplicates', repodir)
1244 if not os.path.isdir(dupdir):
1245 os.makedirs(dupdir, exist_ok=True)
1246 dupfile = os.path.join('duplicates', std_long_name)
1247 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1248 os.rename(apkfile, dupfile)
1249 return True, None, False
1251 os.rename(apkfile, std_long_name)
1252 apkfile = std_long_name
1254 os.rename(apkfile, std_short_name)
1255 apkfile = std_short_name
1256 apkfilename = apkfile[len(repodir) + 1:]
1258 apk['apkName'] = apkfilename
1259 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1260 if os.path.exists(os.path.join(repodir, srcfilename)):
1261 apk['srcname'] = srcfilename
1263 # verify the jar signature is correct, allow deprecated
1264 # algorithms only if the APK is in the archive.
1266 if not common.verify_apk_signature(apkfile):
1267 if repodir == 'archive' or allow_disabled_algorithms:
1268 if common.verify_old_apk_signature(apkfile):
1269 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1277 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1278 move_apk_between_sections(repodir, 'archive', apk)
1280 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1281 return True, None, False
1283 apkzip = zipfile.ZipFile(apkfile, 'r')
1285 # if an APK has files newer than the system time, suggest updating
1286 # the system clock. This is useful for offline systems, used for
1287 # signing, which do not have another source of clock sync info. It
1288 # has to be more than 24 hours newer because ZIP/APK files do not
1289 # store timezone info
1290 manifest = apkzip.getinfo('AndroidManifest.xml')
1291 if manifest.date_time[1] == 0: # month can't be zero
1292 logging.debug('AndroidManifest.xml has no date')
1294 dt_obj = datetime(*manifest.date_time)
1295 checkdt = dt_obj - timedelta(1)
1296 if datetime.today() < checkdt:
1297 logging.warning('System clock is older than manifest in: '
1299 + '\nSet clock to that time using:\n'
1300 + 'sudo date -s "' + str(dt_obj) + '"')
1302 # extract icons from APK zip file
1303 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1305 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1307 apkzip.close() # ensure that APK zip file gets closed
1309 # resize existing icons for densities missing in the APK
1310 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1312 if use_date_from_apk and manifest.date_time[1] != 0:
1313 default_date_param = datetime(*manifest.date_time)
1315 default_date_param = None
1317 # Record in known apks, getting the added date at the same time..
1318 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1319 default_date=default_date_param)
1321 apk['added'] = added
1323 apkcache[apkfilename] = apk
1326 return False, apk, cachechanged
1329 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1330 """Processes the apks in the given repo directory.
1332 This also extracts the icons.
1334 :param apkcache: current apk cache information
1335 :param repodir: repo directory to scan
1336 :param knownapks: known apks info
1337 :param use_date_from_apk: use date from APK (instead of current date)
1338 for newly added APKs
1339 :returns: (apks, cachechanged) where apks is a list of apk information,
1340 and cachechanged is True if the apkcache got changed.
1343 cachechanged = False
1345 for icon_dir in get_all_icon_dirs(repodir):
1346 if os.path.exists(icon_dir):
1348 shutil.rmtree(icon_dir)
1349 os.makedirs(icon_dir)
1351 os.makedirs(icon_dir)
1354 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1355 apkfilename = apkfile[len(repodir) + 1:]
1356 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1357 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1358 use_date_from_apk, ada, True)
1362 cachechanged = cachechanged or cachethis
1364 return apks, cachechanged
1367 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1369 Extracts icons from the given APK zip in various densities,
1370 saves them into given repo directory
1371 and stores their names in the APK metadata dictionary.
1373 :param icon_filename: A string representing the icon's file name
1374 :param apk: A populated dictionary containing APK metadata.
1375 Needs to have 'icons_src' key
1376 :param apkzip: An opened zipfile.ZipFile of the APK file
1377 :param repo_dir: The directory of the APK's repository
1378 :return: A list of icon densities that are missing
1380 empty_densities = []
1381 for density in screen_densities:
1382 if density not in apk['icons_src']:
1383 empty_densities.append(density)
1385 icon_src = apk['icons_src'][density]
1386 icon_dir = get_icon_dir(repo_dir, density)
1387 icon_dest = os.path.join(icon_dir, icon_filename)
1389 # Extract the icon files per density
1390 if icon_src.endswith('.xml'):
1391 png = os.path.basename(icon_src)[:-4] + '.png'
1392 for f in apkzip.namelist():
1394 m = re.match(r'res/drawable-(x*[hlm]dpi).*/', f)
1395 if m and screen_resolutions[m.group(1)] == density:
1398 with open(icon_dest, 'wb') as f:
1399 f.write(get_icon_bytes(apkzip, icon_src))
1400 apk['icons'][density] = icon_filename
1401 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1402 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1403 del apk['icons_src'][density]
1404 empty_densities.append(density)
1406 if '-1' in apk['icons_src']:
1407 icon_src = apk['icons_src']['-1']
1408 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1409 with open(icon_path, 'wb') as f:
1410 f.write(get_icon_bytes(apkzip, icon_src))
1412 im = Image.open(icon_path)
1413 dpi = px_to_dpi(im.size[0])
1414 for density in screen_densities:
1415 if density in apk['icons']:
1417 if density == screen_densities[-1] or dpi >= int(density):
1418 apk['icons'][density] = icon_filename
1419 shutil.move(icon_path,
1420 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1421 empty_densities.remove(density)
1423 except Exception as e:
1424 logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1427 apk['icon'] = icon_filename
1429 return empty_densities
1432 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1434 Resize existing icons for densities missing in the APK to ensure all densities are available
1436 :param empty_densities: A list of icon densities that are missing
1437 :param icon_filename: A string representing the icon's file name
1438 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1439 :param repo_dir: The directory of the APK's repository
1441 # First try resizing down to not lose quality
1443 for density in screen_densities:
1444 if density not in empty_densities:
1445 last_density = density
1447 if last_density is None:
1449 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1451 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1452 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1455 fp = open(last_icon_path, 'rb')
1458 size = dpi_to_px(density)
1460 im.thumbnail((size, size), Image.ANTIALIAS)
1461 im.save(icon_path, "PNG")
1462 empty_densities.remove(density)
1463 except Exception as e:
1464 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1469 # Then just copy from the highest resolution available
1471 for density in reversed(screen_densities):
1472 if density not in empty_densities:
1473 last_density = density
1476 if last_density is None:
1480 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1481 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1483 empty_densities.remove(density)
1485 for density in screen_densities:
1486 icon_dir = get_icon_dir(repo_dir, density)
1487 icon_dest = os.path.join(icon_dir, icon_filename)
1488 resize_icon(icon_dest, density)
1490 # Copy from icons-mdpi to icons since mdpi is the baseline density
1491 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1492 if os.path.isfile(baseline):
1493 apk['icons']['0'] = icon_filename
1494 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1497 def apply_info_from_latest_apk(apps, apks):
1499 Some information from the apks needs to be applied up to the application level.
1500 When doing this, we use the info from the most recent version's apk.
1501 We deal with figuring out when the app was added and last updated at the same time.
1503 for appid, app in apps.items():
1504 bestver = UNSET_VERSION_CODE
1506 if apk['packageName'] == appid:
1507 if apk['versionCode'] > bestver:
1508 bestver = apk['versionCode']
1512 if not app.added or apk['added'] < app.added:
1513 app.added = apk['added']
1514 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1515 app.lastUpdated = apk['added']
1518 logging.debug("Don't know when " + appid + " was added")
1519 if not app.lastUpdated:
1520 logging.debug("Don't know when " + appid + " was last updated")
1522 if bestver == UNSET_VERSION_CODE:
1524 if app.Name is None:
1525 app.Name = app.AutoName or appid
1527 logging.debug("Application " + appid + " has no packages")
1529 if app.Name is None:
1530 app.Name = bestapk['name']
1531 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1532 if app.CurrentVersionCode is None:
1533 app.CurrentVersionCode = str(bestver)
1536 def make_categories_txt(repodir, categories):
1537 '''Write a category list in the repo to allow quick access'''
1539 for cat in sorted(categories):
1540 catdata += cat + '\n'
1541 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1545 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1547 def filter_apk_list_sorted(apk_list):
1549 for apk in apk_list:
1550 if apk['packageName'] == appid:
1553 # Sort the apk list by version code. First is highest/newest.
1554 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1556 for appid, app in apps.items():
1558 if app.ArchivePolicy:
1559 keepversions = int(app.ArchivePolicy[:-9])
1561 keepversions = defaultkeepversions
1563 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1564 .format(appid, len(apks), keepversions, len(archapks)))
1566 current_app_apks = filter_apk_list_sorted(apks)
1567 if len(current_app_apks) > keepversions:
1568 # Move back the ones we don't want.
1569 for apk in current_app_apks[keepversions:]:
1570 move_apk_between_sections(repodir, archivedir, apk)
1571 archapks.append(apk)
1574 current_app_archapks = filter_apk_list_sorted(archapks)
1575 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1577 # Move forward the ones we want again, except DisableAlgorithm
1578 for apk in current_app_archapks:
1579 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1580 move_apk_between_sections(archivedir, repodir, apk)
1581 archapks.remove(apk)
1584 if kept == keepversions:
1588 def move_apk_between_sections(from_dir, to_dir, apk):
1589 """move an APK from repo to archive or vice versa"""
1591 def _move_file(from_dir, to_dir, filename, ignore_missing):
1592 from_path = os.path.join(from_dir, filename)
1593 if ignore_missing and not os.path.exists(from_path):
1595 to_path = os.path.join(to_dir, filename)
1596 if not os.path.exists(to_dir):
1598 shutil.move(from_path, to_path)
1600 if from_dir == to_dir:
1603 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1604 _move_file(from_dir, to_dir, apk['apkName'], False)
1605 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1606 for density in all_screen_densities:
1607 from_icon_dir = get_icon_dir(from_dir, density)
1608 to_icon_dir = get_icon_dir(to_dir, density)
1609 if density not in apk['icons']:
1611 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1612 if 'srcname' in apk:
1613 _move_file(from_dir, to_dir, apk['srcname'], False)
1616 def add_apks_to_per_app_repos(repodir, apks):
1617 apks_per_app = dict()
1619 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1620 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1621 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1622 apks_per_app[apk['packageName']] = apk
1624 if not os.path.exists(apk['per_app_icons']):
1625 logging.info('Adding new repo for only ' + apk['packageName'])
1626 os.makedirs(apk['per_app_icons'])
1628 apkpath = os.path.join(repodir, apk['apkName'])
1629 shutil.copy(apkpath, apk['per_app_repo'])
1630 apksigpath = apkpath + '.sig'
1631 if os.path.exists(apksigpath):
1632 shutil.copy(apksigpath, apk['per_app_repo'])
1633 apkascpath = apkpath + '.asc'
1634 if os.path.exists(apkascpath):
1635 shutil.copy(apkascpath, apk['per_app_repo'])
1644 global config, options
1646 # Parse command line...
1647 parser = ArgumentParser()
1648 common.setup_global_opts(parser)
1649 parser.add_argument("--create-key", action="store_true", default=False,
1650 help="Create a repo signing key in a keystore")
1651 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1652 help="Create skeleton metadata files that are missing")
1653 parser.add_argument("--delete-unknown", action="store_true", default=False,
1654 help="Delete APKs and/or OBBs without metadata from the repo")
1655 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1656 help="Report on build data status")
1657 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1658 help="Interactively ask about things that need updating.")
1659 parser.add_argument("-I", "--icons", action="store_true", default=False,
1660 help="Resize all the icons exceeding the max pixel size and exit")
1661 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1662 help="Specify editor to use in interactive mode. Default " +
1663 "is /etc/alternatives/editor")
1664 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1665 help="Update the wiki")
1666 parser.add_argument("--pretty", action="store_true", default=False,
1667 help="Produce human-readable index.xml")
1668 parser.add_argument("--clean", action="store_true", default=False,
1669 help="Clean update - don't uses caches, reprocess all apks")
1670 parser.add_argument("--nosign", action="store_true", default=False,
1671 help="When configured for signed indexes, create only unsigned indexes at this stage")
1672 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1673 help="Use date from apk instead of current time for newly added apks")
1674 parser.add_argument("--rename-apks", action="store_true", default=False,
1675 help="Rename APK files that do not match package.name_123.apk")
1676 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1677 help="Include APKs that are signed with disabled algorithms like MD5")
1678 metadata.add_metadata_arguments(parser)
1679 options = parser.parse_args()
1680 metadata.warnings_action = options.W
1682 config = common.read_config(options)
1684 if not ('jarsigner' in config and 'keytool' in config):
1685 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1688 if config['archive_older'] != 0:
1689 repodirs.append('archive')
1690 if not os.path.exists('archive'):
1694 resize_all_icons(repodirs)
1697 if options.rename_apks:
1698 options.clean = True
1700 # check that icons exist now, rather than fail at the end of `fdroid update`
1701 for k in ['repo_icon', 'archive_icon']:
1703 if not os.path.exists(config[k]):
1704 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1707 # if the user asks to create a keystore, do it now, reusing whatever it can
1708 if options.create_key:
1709 if os.path.exists(config['keystore']):
1710 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1711 logging.critical("\t'" + config['keystore'] + "'")
1714 if 'repo_keyalias' not in config:
1715 config['repo_keyalias'] = socket.getfqdn()
1716 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1717 if 'keydname' not in config:
1718 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1719 common.write_to_config(config, 'keydname', config['keydname'])
1720 if 'keystore' not in config:
1721 config['keystore'] = common.default_config['keystore']
1722 common.write_to_config(config, 'keystore', config['keystore'])
1724 password = common.genpassword()
1725 if 'keystorepass' not in config:
1726 config['keystorepass'] = password
1727 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1728 if 'keypass' not in config:
1729 config['keypass'] = password
1730 common.write_to_config(config, 'keypass', config['keypass'])
1731 common.genkeystore(config)
1734 apps = metadata.read_metadata()
1736 # Generate a list of categories...
1738 for app in apps.values():
1739 categories.update(app.Categories)
1741 # Read known apks data (will be updated and written back when we've finished)
1742 knownapks = common.KnownApks()
1745 apkcache = get_cache()
1747 # Delete builds for disabled apps
1748 delete_disabled_builds(apps, apkcache, repodirs)
1750 # Scan all apks in the main repo
1751 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1753 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1754 options.use_date_from_apk)
1755 cachechanged = cachechanged or fcachechanged
1757 # Generate warnings for apk's with no metadata (or create skeleton
1758 # metadata files, if requested on the command line)
1761 if apk['packageName'] not in apps:
1762 if options.create_metadata:
1763 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1764 app = metadata.App()
1766 app.Name = apk['name']
1767 app.Summary = apk['name']
1769 logging.warn(apk['packageName'] + ' does not have a name! Using package name instead.')
1770 app.Name = apk['packageName']
1771 app.Summary = apk['packageName']
1772 app.CurrentVersionCode = 2147483647 # Java's Integer.MAX_VALUE
1773 app.Categories = [os.path.basename(os.path.dirname(os.getcwd()))]
1774 metadata.write_yaml(f, app)
1775 logging.info("Generated skeleton metadata for " + apk['packageName'])
1778 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1779 if options.delete_unknown:
1780 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1781 rmf = os.path.join(repodirs[0], apk['apkName'])
1782 if not os.path.exists(rmf):
1783 logging.error("Could not find {0} to remove it".format(rmf))
1787 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1789 # update the metadata with the newly created ones included
1791 apps = metadata.read_metadata()
1793 copy_triple_t_store_metadata(apps)
1794 insert_obbs(repodirs[0], apps, apks)
1795 insert_localized_app_metadata(apps)
1796 translate_per_build_anti_features(apps, apks)
1798 # Scan the archive repo for apks as well
1799 if len(repodirs) > 1:
1800 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1806 # Apply information from latest apks to the application and update dates
1807 apply_info_from_latest_apk(apps, apks + archapks)
1809 # Sort the app list by name, then the web site doesn't have to by default.
1810 # (we had to wait until we'd scanned the apks to do this, because mostly the
1811 # name comes from there!)
1812 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1814 # APKs are placed into multiple repos based on the app package, providing
1815 # per-app subscription feeds for nightly builds and things like it
1816 if config['per_app_repos']:
1817 add_apks_to_per_app_repos(repodirs[0], apks)
1818 for appid, app in apps.items():
1819 repodir = os.path.join(appid, 'fdroid', 'repo')
1821 appdict[appid] = app
1822 if os.path.isdir(repodir):
1823 index.make(appdict, [appid], apks, repodir, False)
1825 logging.info('Skipping index generation for ' + appid)
1828 if len(repodirs) > 1:
1829 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1831 # Make the index for the main repo...
1832 index.make(apps, sortedids, apks, repodirs[0], False)
1833 make_categories_txt(repodirs[0], categories)
1835 # If there's an archive repo, make the index for it. We already scanned it
1837 if len(repodirs) > 1:
1838 index.make(apps, sortedids, archapks, repodirs[1], True)
1840 git_remote = config.get('binary_transparency_remote')
1841 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1843 btlog.make_binary_transparency_log(repodirs)
1845 if config['update_stats']:
1846 # Update known apks info...
1847 knownapks.writeifchanged()
1849 # Generate latest apps data for widget
1850 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1852 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1854 appid = line.rstrip()
1855 data += appid + "\t"
1857 data += app.Name + "\t"
1858 if app.icon is not None:
1859 data += app.icon + "\t"
1860 data += app.License + "\n"
1861 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1865 write_cache(apkcache)
1867 # Update the wiki...
1869 update_wiki(apps, sortedids, apks + archapks)
1871 logging.info("Finished.")
1874 if __name__ == "__main__":