3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime
33 from argparse import ArgumentParser
36 from binascii import hexlify
44 from . import metadata
45 from .common import SdkToolsPopen
46 from .exception import BuildException, FDroidException
50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
51 UNSET_VERSION_CODE = -0x100000000
53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
60 APK_PERMISSION_PAT = \
61 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
64 screen_densities = ['640', '480', '320', '240', '160', '120']
65 screen_resolutions = {
77 all_screen_densities = ['0'] + screen_densities
79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
85 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
88 def dpi_to_px(density):
89 return (int(density) * 48) / 160
93 return (int(px) * 160) / 48
96 def get_icon_dir(repodir, density):
98 return os.path.join(repodir, "icons")
99 return os.path.join(repodir, "icons-%s" % density)
102 def get_icon_dirs(repodir):
103 for density in screen_densities:
104 yield get_icon_dir(repodir, density)
107 def get_all_icon_dirs(repodir):
108 for density in all_screen_densities:
109 yield get_icon_dir(repodir, density)
112 def update_wiki(apps, sortedids, apks):
115 :param apps: fully populated list of all applications
116 :param apks: all apks, except...
118 logging.info("Updating wiki")
120 wikiredircat = 'App Redirects'
122 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
123 path=config['wiki_path'])
124 site.login(config['wiki_user'], config['wiki_password'])
126 generated_redirects = {}
128 for appid in sortedids:
129 app = metadata.App(apps[appid])
133 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
135 for af in sorted(app.AntiFeatures):
136 wikidata += '{{AntiFeature|' + af + '}}\n'
141 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
144 app.added.strftime('%Y-%m-%d') if app.added else '',
145 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
161 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
163 wikidata += app.Summary
164 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
166 wikidata += "=Description=\n"
167 wikidata += metadata.description_wiki(app.Description) + "\n"
169 wikidata += "=Maintainer Notes=\n"
170 if app.MaintainerNotes:
171 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
172 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
174 # Get a list of all packages for this application...
176 gotcurrentver = False
180 if apk['packageName'] == appid:
181 if str(apk['versionCode']) == app.CurrentVersionCode:
184 # Include ones we can't build, as a special case...
185 for build in app.builds:
187 if build.versionCode == app.CurrentVersionCode:
189 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
190 apklist.append({'versionCode': int(build.versionCode),
191 'versionName': build.versionName,
192 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
197 if apk['versionCode'] == int(build.versionCode):
202 apklist.append({'versionCode': int(build.versionCode),
203 'versionName': build.versionName,
204 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
206 if app.CurrentVersionCode == '0':
208 # Sort with most recent first...
209 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
211 wikidata += "=Versions=\n"
212 if len(apklist) == 0:
213 wikidata += "We currently have no versions of this app available."
214 elif not gotcurrentver:
215 wikidata += "We don't have the current version of this app."
217 wikidata += "We have the current version of this app."
218 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
219 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
220 if len(app.NoSourceSince) > 0:
221 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
222 if len(app.CurrentVersion) > 0:
223 wikidata += "The current (recommended) version is " + app.CurrentVersion
224 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
227 wikidata += "==" + apk['versionName'] + "==\n"
229 if 'buildproblem' in apk:
230 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
233 wikidata += "This version is built and signed by "
235 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
237 wikidata += "the original developer.\n\n"
238 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
240 wikidata += '\n[[Category:' + wikicat + ']]\n'
241 if len(app.NoSourceSince) > 0:
242 wikidata += '\n[[Category:Apps missing source code]]\n'
243 if validapks == 0 and not app.Disabled:
244 wikidata += '\n[[Category:Apps with no packages]]\n'
245 if cantupdate and not app.Disabled:
246 wikidata += "\n[[Category:Apps we cannot update]]\n"
247 if buildfails and not app.Disabled:
248 wikidata += "\n[[Category:Apps with failing builds]]\n"
249 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
250 wikidata += '\n[[Category:Apps to Update]]\n'
252 wikidata += '\n[[Category:Apps that are disabled]]\n'
253 if app.UpdateCheckMode == 'None' and not app.Disabled:
254 wikidata += '\n[[Category:Apps with no update check]]\n'
255 for appcat in app.Categories:
256 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
258 # We can't have underscores in the page name, even if they're in
259 # the package ID, because MediaWiki messes with them...
260 pagename = appid.replace('_', ' ')
262 # Drop a trailing newline, because mediawiki is going to drop it anyway
263 # and it we don't we'll think the page has changed when it hasn't...
264 if wikidata.endswith('\n'):
265 wikidata = wikidata[:-1]
267 generated_pages[pagename] = wikidata
269 # Make a redirect from the name to the ID too, unless there's
270 # already an existing page with the name and it isn't a redirect.
272 apppagename = app.Name.replace('_', ' ')
273 apppagename = apppagename.replace('{', '')
274 apppagename = apppagename.replace('}', ' ')
275 apppagename = apppagename.replace(':', ' ')
276 apppagename = apppagename.replace('[', ' ')
277 apppagename = apppagename.replace(']', ' ')
278 # Drop double spaces caused mostly by replacing ':' above
279 apppagename = apppagename.replace(' ', ' ')
280 for expagename in site.allpages(prefix=apppagename,
281 filterredir='nonredirects',
283 if expagename == apppagename:
285 # Another reason not to make the redirect page is if the app name
286 # is the same as it's ID, because that will overwrite the real page
287 # with an redirect to itself! (Although it seems like an odd
288 # scenario this happens a lot, e.g. where there is metadata but no
289 # builds or binaries to extract a name from.
290 if apppagename == pagename:
293 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
295 for tcat, genp in [(wikicat, generated_pages),
296 (wikiredircat, generated_redirects)]:
297 catpages = site.Pages['Category:' + tcat]
299 for page in catpages:
300 existingpages.append(page.name)
301 if page.name in genp:
302 pagetxt = page.edit()
303 if pagetxt != genp[page.name]:
304 logging.debug("Updating modified page " + page.name)
305 page.save(genp[page.name], summary='Auto-updated')
307 logging.debug("Page " + page.name + " is unchanged")
309 logging.warn("Deleting page " + page.name)
310 page.delete('No longer published')
311 for pagename, text in genp.items():
312 logging.debug("Checking " + pagename)
313 if pagename not in existingpages:
314 logging.debug("Creating page " + pagename)
316 newpage = site.Pages[pagename]
317 newpage.save(text, summary='Auto-created')
318 except Exception as e:
319 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
321 # Purge server cache to ensure counts are up to date
322 site.pages['Repository Maintenance'].purge()
325 def delete_disabled_builds(apps, apkcache, repodirs):
326 """Delete disabled build outputs.
328 :param apps: list of all applications, as per metadata.read_metadata
329 :param apkcache: current apk cache information
330 :param repodirs: the repo directories to process
332 for appid, app in apps.items():
333 for build in app['builds']:
334 if not build.disable:
336 apkfilename = common.get_release_filename(app, build)
337 iconfilename = "%s.%s.png" % (
340 for repodir in repodirs:
342 os.path.join(repodir, apkfilename),
343 os.path.join(repodir, apkfilename + '.asc'),
344 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
346 for density in all_screen_densities:
347 repo_dir = get_icon_dir(repodir, density)
348 files.append(os.path.join(repo_dir, iconfilename))
351 if os.path.exists(f):
352 logging.info("Deleting disabled build output " + f)
354 if apkfilename in apkcache:
355 del apkcache[apkfilename]
358 def resize_icon(iconpath, density):
360 if not os.path.isfile(iconpath):
365 fp = open(iconpath, 'rb')
367 size = dpi_to_px(density)
369 if any(length > size for length in im.size):
371 im.thumbnail((size, size), Image.ANTIALIAS)
372 logging.debug("%s was too large at %s - new size is %s" % (
373 iconpath, oldsize, im.size))
374 im.save(iconpath, "PNG")
376 except Exception as e:
377 logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
384 def resize_all_icons(repodirs):
385 """Resize all icons that exceed the max size
387 :param repodirs: the repo directories to process
389 for repodir in repodirs:
390 for density in screen_densities:
391 icon_dir = get_icon_dir(repodir, density)
392 icon_glob = os.path.join(icon_dir, '*.png')
393 for iconpath in glob.glob(icon_glob):
394 resize_icon(iconpath, density)
398 """ Get the signing certificate of an apk. To get the same md5 has that
399 Android gets, we encode the .RSA certificate in a specific format and pass
400 it hex-encoded to the md5 digest algorithm.
402 :param apkpath: path to the apk
403 :returns: A string containing the md5 of the signature of the apk or None
404 if an error occurred.
407 with zipfile.ZipFile(apkpath, 'r') as apk:
408 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
411 logging.error(_("No signing certificates found in {path}").format(path=apkpath))
414 logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
417 cert = apk.read(certs[0])
419 cert_encoded = common.get_certificate(cert)
421 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
424 def get_cache_file():
425 return os.path.join('tmp', 'apkcache')
429 """Get the cached dict of the APK index
431 Gather information about all the apk files in the repo directory,
432 using cached data if possible. Some of the index operations take a
433 long time, like calculating the SHA-256 and verifying the APK
436 The cache is invalidated if the metadata version is different, or
437 the 'allow_disabled_algorithms' config/option is different. In
438 those cases, there is no easy way to know what has changed from
439 the cache, so just rerun the whole thing.
444 apkcachefile = get_cache_file()
445 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
446 if not options.clean and os.path.exists(apkcachefile):
447 with open(apkcachefile, 'rb') as cf:
448 apkcache = pickle.load(cf, encoding='utf-8')
449 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
450 or apkcache.get('allow_disabled_algorithms') != ada:
455 apkcache["METADATA_VERSION"] = METADATA_VERSION
456 apkcache['allow_disabled_algorithms'] = ada
461 def write_cache(apkcache):
462 apkcachefile = get_cache_file()
463 cache_path = os.path.dirname(apkcachefile)
464 if not os.path.exists(cache_path):
465 os.makedirs(cache_path)
466 with open(apkcachefile, 'wb') as cf:
467 pickle.dump(apkcache, cf)
470 def get_icon_bytes(apkzip, iconsrc):
471 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
473 return apkzip.read(iconsrc)
475 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
478 def sha256sum(filename):
479 '''Calculate the sha256 of the given file'''
480 sha = hashlib.sha256()
481 with open(filename, 'rb') as f:
487 return sha.hexdigest()
490 def has_known_vulnerability(filename):
491 """checks for known vulnerabilities in the APK
493 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
494 version. Google also enforces this:
495 https://support.google.com/faqs/answer/6376725?hl=en
497 Checks whether there are more than one classes.dex or AndroidManifest.xml
498 files, which is invalid and an essential part of the "Master Key" attack.
499 http://www.saurik.com/id/17
501 Janus is similar to Master Key but is perhaps easier to scan for.
502 https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
507 # statically load this pattern
508 if not hasattr(has_known_vulnerability, "pattern"):
509 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
511 with open(filename.encode(), 'rb') as fp:
513 if first4 != b'\x50\x4b\x03\x04':
514 raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
515 .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
516 + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
519 with zipfile.ZipFile(filename) as zf:
520 for name in zf.namelist():
521 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
524 chunk = lib.read(4096)
527 m = has_known_vulnerability.pattern.search(chunk)
529 version = m.group(1).decode('ascii')
530 if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
531 or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
532 or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
533 logging.debug(_('"{path}" contains recent {name} ({version})')
534 .format(path=filename, name=name, version=version))
536 logging.warning(_('"{path}" contains outdated {name} ({version})')
537 .format(path=filename, name=name, version=version))
540 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
541 if name in files_in_apk:
542 logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
543 .format(apkfilename=filename, name=name))
545 files_in_apk.add(name)
549 def insert_obbs(repodir, apps, apks):
550 """Scans the .obb files in a given repo directory and adds them to the
551 relevant APK instances. OBB files have versionCodes like APK
552 files, and they are loosely associated. If there is an OBB file
553 present, then any APK with the same or higher versionCode will use
554 that OBB file. There are two OBB types: main and patch, each APK
555 can only have only have one of each.
557 https://developer.android.com/google/play/expansion-files.html
559 :param repodir: repo directory to scan
560 :param apps: list of current, valid apps
561 :param apks: current information on all APKs
565 def obbWarnDelete(f, msg):
566 logging.warning(msg + ' ' + f)
567 if options.delete_unknown:
568 logging.error(_("Deleting unknown file: {path}").format(path=f))
572 java_Integer_MIN_VALUE = -pow(2, 31)
573 currentPackageNames = apps.keys()
574 for f in glob.glob(os.path.join(repodir, '*.obb')):
575 obbfile = os.path.basename(f)
576 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
577 chunks = obbfile.split('.')
578 if chunks[0] != 'main' and chunks[0] != 'patch':
579 obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
581 if not re.match(r'^-?[0-9]+$', chunks[1]):
582 obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
583 .format(name=chunks[0]))
585 versionCode = int(chunks[1])
586 packagename = ".".join(chunks[2:-1])
588 highestVersionCode = java_Integer_MIN_VALUE
589 if packagename not in currentPackageNames:
590 obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
593 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
594 highestVersionCode = apk['versionCode']
595 if versionCode > highestVersionCode:
596 obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
597 .format(integer=str(versionCode)))
599 obbsha256 = sha256sum(f)
600 obbs.append((packagename, versionCode, obbfile, obbsha256))
603 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
604 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
605 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
606 apk['obbMainFile'] = obbfile
607 apk['obbMainFileSha256'] = obbsha256
608 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
609 apk['obbPatchFile'] = obbfile
610 apk['obbPatchFileSha256'] = obbsha256
611 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
615 def translate_per_build_anti_features(apps, apks):
616 """Grab the anti-features list from the build metadata
618 For most Anti-Features, they are really most applicable per-APK,
619 not for an app. An app can fix a vulnerability, add/remove
620 tracking, etc. This reads the 'antifeatures' list from the Build
621 entries in the fdroiddata metadata file, then transforms it into
622 the 'antiFeatures' list of unique items for the index.
624 The field key is all lower case in the metadata file to match the
625 rest of the Build fields. It is 'antiFeatures' camel case in the
626 implementation, index, and fdroidclient since it is translated
627 from the build 'antifeatures' field, not directly included.
631 antiFeatures = dict()
632 for packageName, app in apps.items():
634 for build in app['builds']:
635 afl = build.get('antifeatures')
637 d[int(build.versionCode)] = afl
639 antiFeatures[packageName] = d
642 d = antiFeatures.get(apk['packageName'])
644 afl = d.get(apk['versionCode'])
646 apk['antiFeatures'].update(afl)
649 def _get_localized_dict(app, locale):
650 '''get the dict to add localized store metadata to'''
651 if 'localized' not in app:
652 app['localized'] = collections.OrderedDict()
653 if locale not in app['localized']:
654 app['localized'][locale] = collections.OrderedDict()
655 return app['localized'][locale]
658 def _set_localized_text_entry(app, locale, key, f):
659 limit = config['char_limits'][key]
660 localized = _get_localized_dict(app, locale)
662 text = fp.read()[:limit]
664 localized[key] = text
667 def _set_author_entry(app, key, f):
668 limit = config['char_limits']['author']
670 text = fp.read()[:limit]
675 def _strip_and_copy_image(inpath, outpath):
676 """Remove any metadata from image and copy it to new path
678 Sadly, image metadata like EXIF can be used to exploit devices.
679 It is not used at all in the F-Droid ecosystem, so its much safer
680 just to remove it entirely. PNG does not have the same kind of
685 if common.has_extension(inpath, 'png'):
686 shutil.copy(inpath, outpath)
688 with open(inpath) as fp:
689 in_image = Image.open(fp)
690 data = list(in_image.getdata())
691 out_image = Image.new(in_image.mode, in_image.size)
692 out_image.putdata(data)
693 out_image.save(outpath, "JPEG", optimize=True)
696 def copy_triple_t_store_metadata(apps):
697 """Include store metadata from the app's source repo
699 The Triple-T Gradle Play Publisher is a plugin that has a standard
700 file layout for all of the metadata and graphics that the Google
701 Play Store accepts. Since F-Droid has the git repo, it can just
702 pluck those files directly. This method reads any text files into
703 the app dict, then copies any graphics into the fdroid repo
706 This needs to be run before insert_localized_app_metadata() so that
707 the graphics files that are copied into the fdroid repo get
710 https://github.com/Triple-T/gradle-play-publisher#upload-images
711 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
715 if not os.path.isdir('build'):
716 return # nothing to do
718 for packageName, app in apps.items():
719 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
720 logging.debug('Triple-T Gradle Play Publisher: ' + d)
721 for root, dirs, files in os.walk(d):
722 segments = root.split('/')
723 locale = segments[-2]
725 if f == 'fulldescription':
726 _set_localized_text_entry(app, locale, 'description',
727 os.path.join(root, f))
729 elif f == 'shortdescription':
730 _set_localized_text_entry(app, locale, 'summary',
731 os.path.join(root, f))
734 _set_localized_text_entry(app, locale, 'name',
735 os.path.join(root, f))
738 _set_localized_text_entry(app, locale, 'video',
739 os.path.join(root, f))
741 elif f == 'whatsnew':
742 _set_localized_text_entry(app, segments[-1], 'whatsNew',
743 os.path.join(root, f))
745 elif f == 'contactEmail':
746 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
748 elif f == 'contactPhone':
749 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
751 elif f == 'contactWebsite':
752 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
755 base, extension = common.get_extension(f)
756 dirname = os.path.basename(root)
757 if extension in ALLOWED_EXTENSIONS \
758 and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
759 if segments[-2] == 'listing':
760 locale = segments[-3]
762 locale = segments[-2]
763 destdir = os.path.join('repo', packageName, locale, dirname)
764 os.makedirs(destdir, mode=0o755, exist_ok=True)
765 sourcefile = os.path.join(root, f)
766 destfile = os.path.join(destdir, os.path.basename(f))
767 logging.debug('copying ' + sourcefile + ' ' + destfile)
768 _strip_and_copy_image(sourcefile, destfile)
771 def insert_localized_app_metadata(apps):
772 """scans standard locations for graphics and localized text
774 Scans for localized description files, store graphics, and
775 screenshot PNG files in statically defined screenshots directory
776 and adds them to the app metadata. The screenshots and graphic
777 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
778 and must be in the following layout:
779 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
781 repo/packageName/locale/featureGraphic.png
782 repo/packageName/locale/phoneScreenshots/1.png
783 repo/packageName/locale/phoneScreenshots/2.png
785 The changelog files must be text files named with the versionCode
786 ending with ".txt" and must be in the following layout:
787 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
789 repo/packageName/locale/changelogs/12345.txt
791 This will scan the each app's source repo then the metadata/ dir
792 for these standard locations of changelog files. If it finds
793 them, they will be added to the dict of all packages, with the
794 versions in the metadata/ folder taking precendence over the what
795 is in the app's source repo.
797 Where "packageName" is the app's packageName and "locale" is the locale
798 of the graphics, e.g. what language they are in, using the IETF RFC5646
799 format (en-US, fr-CA, es-MX, etc).
801 This will also scan the app's git for a fastlane folder, and the
802 metadata/ folder and the apps' source repos for standard locations
803 of graphic and screenshot files. If it finds them, it will copy
804 them into the repo. The fastlane files follow this pattern:
805 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
809 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
810 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
811 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
812 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
814 for srcd in sorted(sourcedirs):
815 if not os.path.isdir(srcd):
817 for root, dirs, files in os.walk(srcd):
818 segments = root.split('/')
819 packageName = segments[1]
820 if packageName not in apps:
821 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
823 locale = segments[-1]
824 destdir = os.path.join('repo', packageName, locale)
826 # flavours specified in build receipt
828 if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
829 and 'gradle' in apps[packageName].builds[-1]:
830 build_flavours = apps[packageName].builds[-1].gradle
832 if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
833 logging.debug("ignoring due to wrong flavour")
837 if f in ('description.txt', 'full_description.txt'):
838 _set_localized_text_entry(apps[packageName], locale, 'description',
839 os.path.join(root, f))
841 elif f in ('summary.txt', 'short_description.txt'):
842 _set_localized_text_entry(apps[packageName], locale, 'summary',
843 os.path.join(root, f))
845 elif f in ('name.txt', 'title.txt'):
846 _set_localized_text_entry(apps[packageName], locale, 'name',
847 os.path.join(root, f))
849 elif f == 'video.txt':
850 _set_localized_text_entry(apps[packageName], locale, 'video',
851 os.path.join(root, f))
853 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
854 locale = segments[-2]
855 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
856 os.path.join(root, f))
859 base, extension = common.get_extension(f)
860 if locale == 'images':
861 locale = segments[-2]
862 destdir = os.path.join('repo', packageName, locale)
863 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
864 os.makedirs(destdir, mode=0o755, exist_ok=True)
865 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
866 _strip_and_copy_image(os.path.join(root, f), destdir)
868 if d in SCREENSHOT_DIRS:
869 if locale == 'images':
870 locale = segments[-2]
871 destdir = os.path.join('repo', packageName, locale)
872 for f in glob.glob(os.path.join(root, d, '*.*')):
873 _ignored, extension = common.get_extension(f)
874 if extension in ALLOWED_EXTENSIONS:
875 screenshotdestdir = os.path.join(destdir, d)
876 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
877 logging.debug('copying ' + f + ' ' + screenshotdestdir)
878 _strip_and_copy_image(f, screenshotdestdir)
880 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
882 if not os.path.isdir(d):
884 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
885 if not os.path.isfile(f):
887 segments = f.split('/')
888 packageName = segments[1]
890 screenshotdir = segments[3]
891 filename = os.path.basename(f)
892 base, extension = common.get_extension(filename)
894 if packageName not in apps:
895 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
896 .format(path=filename, name=packageName))
898 graphics = _get_localized_dict(apps[packageName], locale)
900 if extension not in ALLOWED_EXTENSIONS:
901 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
902 elif base in GRAPHIC_NAMES:
903 # there can only be zero or one of these per locale
904 graphics[base] = filename
905 elif screenshotdir in SCREENSHOT_DIRS:
906 # there can any number of these per locale
907 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
908 if screenshotdir not in graphics:
909 graphics[screenshotdir] = []
910 graphics[screenshotdir].append(filename)
912 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
915 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
916 """Scan a repo for all files with an extension except APK/OBB
918 :param apkcache: current cached info about all repo files
919 :param repodir: repo directory to scan
920 :param knownapks: list of all known files, as per metadata.read_metadata
921 :param use_date_from_file: use date from file (instead of current date)
922 for newly added files
927 repodir = repodir.encode('utf-8')
928 for name in os.listdir(repodir):
929 file_extension = common.get_file_extension(name)
930 if file_extension == 'apk' or file_extension == 'obb':
932 filename = os.path.join(repodir, name)
933 name_utf8 = name.decode('utf-8')
934 if filename.endswith(b'_src.tar.gz'):
935 logging.debug(_('skipping source tarball: {path}')
936 .format(path=filename.decode('utf-8')))
938 if not common.is_repo_file(filename):
940 stat = os.stat(filename)
941 if stat.st_size == 0:
942 raise FDroidException(_('{path} is zero size!')
943 .format(path=filename))
945 shasum = sha256sum(filename)
948 repo_file = apkcache[name]
949 # added time is cached as tuple but used here as datetime instance
950 if 'added' in repo_file:
951 a = repo_file['added']
952 if isinstance(a, datetime):
953 repo_file['added'] = a
955 repo_file['added'] = datetime(*a[:6])
956 if repo_file.get('hash') == shasum:
957 logging.debug(_("Reading {apkfilename} from cache")
958 .format(apkfilename=name_utf8))
961 logging.debug(_("Ignoring stale cache data for {apkfilename}")
962 .format(apkfilename=name_utf8))
965 logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
966 repo_file = collections.OrderedDict()
967 repo_file['name'] = os.path.splitext(name_utf8)[0]
968 # TODO rename apkname globally to something more generic
969 repo_file['apkName'] = name_utf8
970 repo_file['hash'] = shasum
971 repo_file['hashType'] = 'sha256'
972 repo_file['versionCode'] = 0
973 repo_file['versionName'] = shasum
974 # the static ID is the SHA256 unless it is set in the metadata
975 repo_file['packageName'] = shasum
977 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
979 repo_file['packageName'] = m.group(1)
980 repo_file['versionCode'] = int(m.group(2))
981 srcfilename = name + b'_src.tar.gz'
982 if os.path.exists(os.path.join(repodir, srcfilename)):
983 repo_file['srcname'] = srcfilename.decode('utf-8')
984 repo_file['size'] = stat.st_size
986 apkcache[name] = repo_file
989 if use_date_from_file:
990 timestamp = stat.st_ctime
991 default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
993 default_date_param = None
995 # Record in knownapks, getting the added date at the same time..
996 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
997 default_date=default_date_param)
999 repo_file['added'] = added
1001 repo_files.append(repo_file)
1003 return repo_files, cachechanged
1006 def scan_apk(apk_file):
1008 Scans an APK file and returns dictionary with metadata of the APK.
1010 Attention: This does *not* verify that the APK signature is correct.
1012 :param apk_file: The (ideally absolute) path to the APK file
1013 :raises BuildException
1014 :return A dict containing APK metadata
1017 'hash': sha256sum(apk_file),
1018 'hashType': 'sha256',
1019 'uses-permission': [],
1020 'uses-permission-sdk-23': [],
1024 'antiFeatures': set(),
1027 if SdkToolsPopen(['aapt', 'version'], output=False):
1028 scan_apk_aapt(apk, apk_file)
1030 scan_apk_androguard(apk, apk_file)
1032 # Get the signature, or rather the signing key fingerprints
1033 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1034 apk['sig'] = getsig(apk_file)
1036 raise BuildException("Failed to get apk signature")
1037 apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1039 if not apk.get('signer'):
1040 raise BuildException("Failed to get apk signing key fingerprint")
1042 # Get size of the APK
1043 apk['size'] = os.path.getsize(apk_file)
1045 if 'minSdkVersion' not in apk:
1046 logging.warning("No SDK version information found in {0}".format(apk_file))
1047 apk['minSdkVersion'] = 1
1049 # Check for known vulnerabilities
1050 if has_known_vulnerability(apk_file):
1051 apk['antiFeatures'].add('KnownVuln')
1056 def scan_apk_aapt(apk, apkfile):
1057 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1058 if p.returncode != 0:
1059 if options.delete_unknown:
1060 if os.path.exists(apkfile):
1061 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1064 logging.error("Could not find {0} to remove it".format(apkfile))
1066 logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1067 raise BuildException(_("Invalid APK"))
1068 for line in p.output.splitlines():
1069 if line.startswith("package:"):
1071 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1072 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1073 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1074 except Exception as e:
1075 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1076 elif line.startswith("application:"):
1077 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1078 # Keep path to non-dpi icon in case we need it
1079 match = re.match(APK_ICON_PAT_NODPI, line)
1081 apk['icons_src']['-1'] = match.group(1)
1082 elif line.startswith("launchable-activity:"):
1083 # Only use launchable-activity as fallback to application
1085 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1086 if '-1' not in apk['icons_src']:
1087 match = re.match(APK_ICON_PAT_NODPI, line)
1089 apk['icons_src']['-1'] = match.group(1)
1090 elif line.startswith("application-icon-"):
1091 match = re.match(APK_ICON_PAT, line)
1093 density = match.group(1)
1094 path = match.group(2)
1095 apk['icons_src'][density] = path
1096 elif line.startswith("sdkVersion:"):
1097 m = re.match(APK_SDK_VERSION_PAT, line)
1099 logging.error(line.replace('sdkVersion:', '')
1100 + ' is not a valid minSdkVersion!')
1102 apk['minSdkVersion'] = m.group(1)
1103 # if target not set, default to min
1104 if 'targetSdkVersion' not in apk:
1105 apk['targetSdkVersion'] = m.group(1)
1106 elif line.startswith("targetSdkVersion:"):
1107 m = re.match(APK_SDK_VERSION_PAT, line)
1109 logging.error(line.replace('targetSdkVersion:', '')
1110 + ' is not a valid targetSdkVersion!')
1112 apk['targetSdkVersion'] = m.group(1)
1113 elif line.startswith("maxSdkVersion:"):
1114 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1115 elif line.startswith("native-code:"):
1116 apk['nativecode'] = []
1117 for arch in line[13:].split(' '):
1118 apk['nativecode'].append(arch[1:-1])
1119 elif line.startswith('uses-permission:'):
1120 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1121 if perm_match['maxSdkVersion']:
1122 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1123 permission = UsesPermission(
1125 perm_match['maxSdkVersion']
1128 apk['uses-permission'].append(permission)
1129 elif line.startswith('uses-permission-sdk-23:'):
1130 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1131 if perm_match['maxSdkVersion']:
1132 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1133 permission_sdk_23 = UsesPermissionSdk23(
1135 perm_match['maxSdkVersion']
1138 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1140 elif line.startswith('uses-feature:'):
1141 feature = re.match(APK_FEATURE_PAT, line).group(1)
1142 # Filter out this, it's only added with the latest SDK tools and
1143 # causes problems for lots of apps.
1144 if feature != "android.hardware.screen.portrait" \
1145 and feature != "android.hardware.screen.landscape":
1146 if feature.startswith("android.feature."):
1147 feature = feature[16:]
1148 apk['features'].add(feature)
1151 def scan_apk_androguard(apk, apkfile):
1153 from androguard.core.bytecodes.apk import APK
1154 apkobject = APK(apkfile)
1155 if apkobject.is_valid_APK():
1156 arsc = apkobject.get_android_resources()
1158 if options.delete_unknown:
1159 if os.path.exists(apkfile):
1160 logging.error(_("Failed to get apk information, deleting {path}")
1161 .format(path=apkfile))
1164 logging.error(_("Could not find {path} to remove it")
1165 .format(path=apkfile))
1167 logging.error(_("Failed to get apk information, skipping {path}")
1168 .format(path=apkfile))
1169 raise BuildException(_("Invalid APK"))
1171 raise FDroidException("androguard library is not installed and aapt not present")
1172 except FileNotFoundError:
1173 logging.error(_("Could not open apk file for analysis"))
1174 raise BuildException(_("Invalid APK"))
1176 apk['packageName'] = apkobject.get_package()
1177 apk['versionCode'] = int(apkobject.get_androidversion_code())
1178 apk['versionName'] = apkobject.get_androidversion_name()
1179 if apk['versionName'][0] == "@":
1180 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1181 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1182 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1183 apk['name'] = apkobject.get_app_name()
1185 if apkobject.get_max_sdk_version() is not None:
1186 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1187 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1188 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1190 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1191 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1193 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1195 for file in apkobject.get_files():
1196 d_re = density_re.match(file)
1198 folder = d_re.group(1).split('-')
1200 resolution = folder[1]
1203 density = screen_resolutions[resolution]
1204 apk['icons_src'][density] = d_re.group(0)
1206 if apk['icons_src'].get('-1') is None:
1207 apk['icons_src']['-1'] = apk['icons_src']['160']
1209 arch_re = re.compile("^lib/(.*)/.*$")
1210 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1212 apk['nativecode'] = []
1213 apk['nativecode'].extend(sorted(list(arch)))
1215 xml = apkobject.get_android_manifest_xml()
1217 for item in xml.getElementsByTagName('uses-permission'):
1218 name = str(item.getAttribute("android:name"))
1219 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1220 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1221 permission = UsesPermission(
1225 apk['uses-permission'].append(permission)
1227 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1228 name = str(item.getAttribute("android:name"))
1229 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1230 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1231 permission_sdk_23 = UsesPermissionSdk23(
1235 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1237 for item in xml.getElementsByTagName('uses-feature'):
1238 feature = str(item.getAttribute("android:name"))
1239 if feature != "android.hardware.screen.portrait" \
1240 and feature != "android.hardware.screen.landscape":
1241 if feature.startswith("android.feature."):
1242 feature = feature[16:]
1243 apk['features'].append(feature)
1246 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1247 allow_disabled_algorithms=False, archive_bad_sig=False):
1248 """Processes the apk with the given filename in the given repo directory.
1250 This also extracts the icons.
1252 :param apkcache: current apk cache information
1253 :param apkfilename: the filename of the apk to scan
1254 :param repodir: repo directory to scan
1255 :param knownapks: known apks info
1256 :param use_date_from_apk: use date from APK (instead of current date)
1257 for newly added APKs
1258 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1259 disabled algorithms in the signature (e.g. MD5)
1260 :param archive_bad_sig: move APKs with a bad signature to the archive
1261 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1262 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1266 apkfile = os.path.join(repodir, apkfilename)
1268 cachechanged = False
1270 if apkfilename in apkcache:
1271 apk = apkcache[apkfilename]
1272 if apk.get('hash') == sha256sum(apkfile):
1273 logging.debug(_("Reading {apkfilename} from cache")
1274 .format(apkfilename=apkfilename))
1277 logging.debug(_("Ignoring stale cache data for {apkfilename}")
1278 .format(apkfilename=apkfilename))
1281 logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1284 apk = scan_apk(apkfile)
1285 except BuildException:
1286 logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1287 .format(apkfilename=apkfilename))
1288 return True, None, False
1290 # Check for debuggable apks...
1291 if common.isApkAndDebuggable(apkfile):
1292 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1294 if options.rename_apks:
1295 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1296 std_short_name = os.path.join(repodir, n)
1297 if apkfile != std_short_name:
1298 if os.path.exists(std_short_name):
1299 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1300 if apkfile != std_long_name:
1301 if os.path.exists(std_long_name):
1302 dupdir = os.path.join('duplicates', repodir)
1303 if not os.path.isdir(dupdir):
1304 os.makedirs(dupdir, exist_ok=True)
1305 dupfile = os.path.join('duplicates', std_long_name)
1306 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1307 os.rename(apkfile, dupfile)
1308 return True, None, False
1310 os.rename(apkfile, std_long_name)
1311 apkfile = std_long_name
1313 os.rename(apkfile, std_short_name)
1314 apkfile = std_short_name
1315 apkfilename = apkfile[len(repodir) + 1:]
1317 apk['apkName'] = apkfilename
1318 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1319 if os.path.exists(os.path.join(repodir, srcfilename)):
1320 apk['srcname'] = srcfilename
1322 # verify the jar signature is correct, allow deprecated
1323 # algorithms only if the APK is in the archive.
1325 if not common.verify_apk_signature(apkfile):
1326 if repodir == 'archive' or allow_disabled_algorithms:
1327 if common.verify_old_apk_signature(apkfile):
1328 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1336 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1337 .format(apkfilename=apkfilename))
1338 move_apk_between_sections(repodir, 'archive', apk)
1340 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1341 .format(apkfilename=apkfilename))
1342 return True, None, False
1344 apkzip = zipfile.ZipFile(apkfile, 'r')
1346 manifest = apkzip.getinfo('AndroidManifest.xml')
1347 if manifest.date_time[1] == 0: # month can't be zero
1348 logging.debug(_('AndroidManifest.xml has no date'))
1350 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1352 # extract icons from APK zip file
1353 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1355 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1357 apkzip.close() # ensure that APK zip file gets closed
1359 # resize existing icons for densities missing in the APK
1360 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1362 if use_date_from_apk and manifest.date_time[1] != 0:
1363 default_date_param = datetime(*manifest.date_time)
1365 default_date_param = None
1367 # Record in known apks, getting the added date at the same time..
1368 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1369 default_date=default_date_param)
1371 apk['added'] = added
1373 apkcache[apkfilename] = apk
1376 return False, apk, cachechanged
1379 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1380 """Processes the apks in the given repo directory.
1382 This also extracts the icons.
1384 :param apkcache: current apk cache information
1385 :param repodir: repo directory to scan
1386 :param knownapks: known apks info
1387 :param use_date_from_apk: use date from APK (instead of current date)
1388 for newly added APKs
1389 :returns: (apks, cachechanged) where apks is a list of apk information,
1390 and cachechanged is True if the apkcache got changed.
1393 cachechanged = False
1395 for icon_dir in get_all_icon_dirs(repodir):
1396 if os.path.exists(icon_dir):
1398 shutil.rmtree(icon_dir)
1399 os.makedirs(icon_dir)
1401 os.makedirs(icon_dir)
1404 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1405 apkfilename = apkfile[len(repodir) + 1:]
1406 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1407 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1408 use_date_from_apk, ada, True)
1412 cachechanged = cachechanged or cachethis
1414 return apks, cachechanged
1417 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1419 Extracts icons from the given APK zip in various densities,
1420 saves them into given repo directory
1421 and stores their names in the APK metadata dictionary.
1423 :param icon_filename: A string representing the icon's file name
1424 :param apk: A populated dictionary containing APK metadata.
1425 Needs to have 'icons_src' key
1426 :param apkzip: An opened zipfile.ZipFile of the APK file
1427 :param repo_dir: The directory of the APK's repository
1428 :return: A list of icon densities that are missing
1430 empty_densities = []
1431 for density in screen_densities:
1432 if density not in apk['icons_src']:
1433 empty_densities.append(density)
1435 icon_src = apk['icons_src'][density]
1436 icon_dir = get_icon_dir(repo_dir, density)
1437 icon_dest = os.path.join(icon_dir, icon_filename)
1439 # Extract the icon files per density
1440 if icon_src.endswith('.xml'):
1441 png = os.path.basename(icon_src)[:-4] + '.png'
1442 for f in apkzip.namelist():
1444 m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1445 if m and screen_resolutions[m.group(2)] == density:
1447 if icon_src.endswith('.xml'):
1448 empty_densities.append(density)
1451 with open(icon_dest, 'wb') as f:
1452 f.write(get_icon_bytes(apkzip, icon_src))
1453 apk['icons'][density] = icon_filename
1454 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1455 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1456 del apk['icons_src'][density]
1457 empty_densities.append(density)
1459 if '-1' in apk['icons_src']:
1460 icon_src = apk['icons_src']['-1']
1461 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1462 with open(icon_path, 'wb') as f:
1463 f.write(get_icon_bytes(apkzip, icon_src))
1465 im = Image.open(icon_path)
1466 dpi = px_to_dpi(im.size[0])
1467 for density in screen_densities:
1468 if density in apk['icons']:
1470 if density == screen_densities[-1] or dpi >= int(density):
1471 apk['icons'][density] = icon_filename
1472 shutil.move(icon_path,
1473 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1474 empty_densities.remove(density)
1476 except Exception as e:
1477 logging.warning(_("Failed reading {path}: {error}")
1478 .format(path=icon_path, error=e))
1481 apk['icon'] = icon_filename
1483 return empty_densities
1486 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1488 Resize existing icons for densities missing in the APK to ensure all densities are available
1490 :param empty_densities: A list of icon densities that are missing
1491 :param icon_filename: A string representing the icon's file name
1492 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1493 :param repo_dir: The directory of the APK's repository
1495 # First try resizing down to not lose quality
1497 for density in screen_densities:
1498 if density not in empty_densities:
1499 last_density = density
1501 if last_density is None:
1503 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1505 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1506 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1509 fp = open(last_icon_path, 'rb')
1512 size = dpi_to_px(density)
1514 im.thumbnail((size, size), Image.ANTIALIAS)
1515 im.save(icon_path, "PNG")
1516 empty_densities.remove(density)
1517 except Exception as e:
1518 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1523 # Then just copy from the highest resolution available
1525 for density in reversed(screen_densities):
1526 if density not in empty_densities:
1527 last_density = density
1530 if last_density is None:
1534 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1535 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1537 empty_densities.remove(density)
1539 for density in screen_densities:
1540 icon_dir = get_icon_dir(repo_dir, density)
1541 icon_dest = os.path.join(icon_dir, icon_filename)
1542 resize_icon(icon_dest, density)
1544 # Copy from icons-mdpi to icons since mdpi is the baseline density
1545 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1546 if os.path.isfile(baseline):
1547 apk['icons']['0'] = icon_filename
1548 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1551 def apply_info_from_latest_apk(apps, apks):
1553 Some information from the apks needs to be applied up to the application level.
1554 When doing this, we use the info from the most recent version's apk.
1555 We deal with figuring out when the app was added and last updated at the same time.
1557 for appid, app in apps.items():
1558 bestver = UNSET_VERSION_CODE
1560 if apk['packageName'] == appid:
1561 if apk['versionCode'] > bestver:
1562 bestver = apk['versionCode']
1566 if not app.added or apk['added'] < app.added:
1567 app.added = apk['added']
1568 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1569 app.lastUpdated = apk['added']
1572 logging.debug("Don't know when " + appid + " was added")
1573 if not app.lastUpdated:
1574 logging.debug("Don't know when " + appid + " was last updated")
1576 if bestver == UNSET_VERSION_CODE:
1578 if app.Name is None:
1579 app.Name = app.AutoName or appid
1581 logging.debug("Application " + appid + " has no packages")
1583 if app.Name is None:
1584 app.Name = bestapk['name']
1585 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1586 if app.CurrentVersionCode is None:
1587 app.CurrentVersionCode = str(bestver)
1590 def make_categories_txt(repodir, categories):
1591 '''Write a category list in the repo to allow quick access'''
1593 for cat in sorted(categories):
1594 catdata += cat + '\n'
1595 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1599 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1601 def filter_apk_list_sorted(apk_list):
1603 for apk in apk_list:
1604 if apk['packageName'] == appid:
1607 # Sort the apk list by version code. First is highest/newest.
1608 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1610 for appid, app in apps.items():
1612 if app.ArchivePolicy:
1613 keepversions = int(app.ArchivePolicy[:-9])
1615 keepversions = defaultkeepversions
1617 logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1618 .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1620 current_app_apks = filter_apk_list_sorted(apks)
1621 if len(current_app_apks) > keepversions:
1622 # Move back the ones we don't want.
1623 for apk in current_app_apks[keepversions:]:
1624 move_apk_between_sections(repodir, archivedir, apk)
1625 archapks.append(apk)
1628 current_app_archapks = filter_apk_list_sorted(archapks)
1629 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1631 # Move forward the ones we want again, except DisableAlgorithm
1632 for apk in current_app_archapks:
1633 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1634 move_apk_between_sections(archivedir, repodir, apk)
1635 archapks.remove(apk)
1638 if kept == keepversions:
1642 def move_apk_between_sections(from_dir, to_dir, apk):
1643 """move an APK from repo to archive or vice versa"""
1645 def _move_file(from_dir, to_dir, filename, ignore_missing):
1646 from_path = os.path.join(from_dir, filename)
1647 if ignore_missing and not os.path.exists(from_path):
1649 to_path = os.path.join(to_dir, filename)
1650 if not os.path.exists(to_dir):
1652 shutil.move(from_path, to_path)
1654 if from_dir == to_dir:
1657 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1658 _move_file(from_dir, to_dir, apk['apkName'], False)
1659 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1660 for density in all_screen_densities:
1661 from_icon_dir = get_icon_dir(from_dir, density)
1662 to_icon_dir = get_icon_dir(to_dir, density)
1663 if density not in apk.get('icons', []):
1665 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1666 if 'srcname' in apk:
1667 _move_file(from_dir, to_dir, apk['srcname'], False)
1670 def add_apks_to_per_app_repos(repodir, apks):
1671 apks_per_app = dict()
1673 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1674 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1675 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1676 apks_per_app[apk['packageName']] = apk
1678 if not os.path.exists(apk['per_app_icons']):
1679 logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1680 os.makedirs(apk['per_app_icons'])
1682 apkpath = os.path.join(repodir, apk['apkName'])
1683 shutil.copy(apkpath, apk['per_app_repo'])
1684 apksigpath = apkpath + '.sig'
1685 if os.path.exists(apksigpath):
1686 shutil.copy(apksigpath, apk['per_app_repo'])
1687 apkascpath = apkpath + '.asc'
1688 if os.path.exists(apkascpath):
1689 shutil.copy(apkascpath, apk['per_app_repo'])
1692 def create_metadata_from_template(apk):
1693 '''create a new metadata file using internal or external template
1695 Generate warnings for apk's with no metadata (or create skeleton
1696 metadata files, if requested on the command line). Though the
1697 template file is YAML, this uses neither pyyaml nor ruamel.yaml
1698 since those impose things on the metadata file made from the
1699 template: field sort order, empty field value, formatting, etc.
1703 if os.path.exists('template.yml'):
1704 with open('template.yml') as f:
1706 if 'name' in apk and apk['name'] != '':
1707 metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1708 r'\1 ' + apk['name'],
1710 flags=re.IGNORECASE | re.MULTILINE)
1712 logging.warning(_('{appid} does not have a name! Using package name instead.')
1713 .format(appid=apk['packageName']))
1714 metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1715 r'\1 ' + apk['packageName'],
1717 flags=re.IGNORECASE | re.MULTILINE)
1718 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1722 app['Categories'] = [os.path.basename(os.getcwd())]
1723 # include some blanks as part of the template
1724 app['AuthorName'] = ''
1727 app['IssueTracker'] = ''
1728 app['SourceCode'] = ''
1729 app['CurrentVersionCode'] = 2147483647 # Java's Integer.MAX_VALUE
1730 if 'name' in apk and apk['name'] != '':
1731 app['Name'] = apk['name']
1733 logging.warning(_('{appid} does not have a name! Using package name instead.')
1734 .format(appid=apk['packageName']))
1735 app['Name'] = apk['packageName']
1736 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1737 yaml.dump(app, f, default_flow_style=False)
1738 logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1747 global config, options
1749 # Parse command line...
1750 parser = ArgumentParser()
1751 common.setup_global_opts(parser)
1752 parser.add_argument("--create-key", action="store_true", default=False,
1753 help=_("Add a repo signing key to an unsigned repo"))
1754 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1755 help=_("Add skeleton metadata files for APKs that are missing them"))
1756 parser.add_argument("--delete-unknown", action="store_true", default=False,
1757 help=_("Delete APKs and/or OBBs without metadata from the repo"))
1758 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1759 help=_("Report on build data status"))
1760 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1761 help=_("Interactively ask about things that need updating."))
1762 parser.add_argument("-I", "--icons", action="store_true", default=False,
1763 help=_("Resize all the icons exceeding the max pixel size and exit"))
1764 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1765 help=_("Specify editor to use in interactive mode. Default " +
1766 "is {path}").format(path='/etc/alternatives/editor'))
1767 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1768 help=_("Update the wiki"))
1769 parser.add_argument("--pretty", action="store_true", default=False,
1770 help=_("Produce human-readable XML/JSON for index files"))
1771 parser.add_argument("--clean", action="store_true", default=False,
1772 help=_("Clean update - don't uses caches, reprocess all APKs"))
1773 parser.add_argument("--nosign", action="store_true", default=False,
1774 help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1775 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1776 help=_("Use date from APK instead of current time for newly added APKs"))
1777 parser.add_argument("--rename-apks", action="store_true", default=False,
1778 help=_("Rename APK files that do not match package.name_123.apk"))
1779 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1780 help=_("Include APKs that are signed with disabled algorithms like MD5"))
1781 metadata.add_metadata_arguments(parser)
1782 options = parser.parse_args()
1783 metadata.warnings_action = options.W
1785 config = common.read_config(options)
1787 if not ('jarsigner' in config and 'keytool' in config):
1788 raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1791 if config['archive_older'] != 0:
1792 repodirs.append('archive')
1793 if not os.path.exists('archive'):
1797 resize_all_icons(repodirs)
1800 if options.rename_apks:
1801 options.clean = True
1803 # check that icons exist now, rather than fail at the end of `fdroid update`
1804 for k in ['repo_icon', 'archive_icon']:
1806 if not os.path.exists(config[k]):
1807 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1808 .format(name=k, path=config[k]))
1811 # if the user asks to create a keystore, do it now, reusing whatever it can
1812 if options.create_key:
1813 if os.path.exists(config['keystore']):
1814 logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1815 logging.critical("\t'" + config['keystore'] + "'")
1818 if 'repo_keyalias' not in config:
1819 config['repo_keyalias'] = socket.getfqdn()
1820 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1821 if 'keydname' not in config:
1822 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1823 common.write_to_config(config, 'keydname', config['keydname'])
1824 if 'keystore' not in config:
1825 config['keystore'] = common.default_config['keystore']
1826 common.write_to_config(config, 'keystore', config['keystore'])
1828 password = common.genpassword()
1829 if 'keystorepass' not in config:
1830 config['keystorepass'] = password
1831 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1832 if 'keypass' not in config:
1833 config['keypass'] = password
1834 common.write_to_config(config, 'keypass', config['keypass'])
1835 common.genkeystore(config)
1838 apps = metadata.read_metadata()
1840 # Generate a list of categories...
1842 for app in apps.values():
1843 categories.update(app.Categories)
1845 # Read known apks data (will be updated and written back when we've finished)
1846 knownapks = common.KnownApks()
1849 apkcache = get_cache()
1851 # Delete builds for disabled apps
1852 delete_disabled_builds(apps, apkcache, repodirs)
1854 # Scan all apks in the main repo
1855 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1857 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1858 options.use_date_from_apk)
1859 cachechanged = cachechanged or fcachechanged
1862 if apk['packageName'] not in apps:
1863 if options.create_metadata:
1864 create_metadata_from_template(apk)
1865 apps = metadata.read_metadata()
1867 msg = _("{apkfilename} ({appid}) has no metadata!") \
1868 .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1869 if options.delete_unknown:
1870 logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1871 .format(apkfilename=apk['apkName']))
1872 rmf = os.path.join(repodirs[0], apk['apkName'])
1873 if not os.path.exists(rmf):
1874 logging.error(_("Could not find {path} to remove it").format(path=rmf))
1878 logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1880 copy_triple_t_store_metadata(apps)
1881 insert_obbs(repodirs[0], apps, apks)
1882 insert_localized_app_metadata(apps)
1883 translate_per_build_anti_features(apps, apks)
1885 # Scan the archive repo for apks as well
1886 if len(repodirs) > 1:
1887 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1893 # Apply information from latest apks to the application and update dates
1894 apply_info_from_latest_apk(apps, apks + archapks)
1896 # Sort the app list by name, then the web site doesn't have to by default.
1897 # (we had to wait until we'd scanned the apks to do this, because mostly the
1898 # name comes from there!)
1899 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1901 # APKs are placed into multiple repos based on the app package, providing
1902 # per-app subscription feeds for nightly builds and things like it
1903 if config['per_app_repos']:
1904 add_apks_to_per_app_repos(repodirs[0], apks)
1905 for appid, app in apps.items():
1906 repodir = os.path.join(appid, 'fdroid', 'repo')
1908 appdict[appid] = app
1909 if os.path.isdir(repodir):
1910 index.make(appdict, [appid], apks, repodir, False)
1912 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1915 if len(repodirs) > 1:
1916 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1918 # Make the index for the main repo...
1919 index.make(apps, sortedids, apks, repodirs[0], False)
1920 make_categories_txt(repodirs[0], categories)
1922 # If there's an archive repo, make the index for it. We already scanned it
1924 if len(repodirs) > 1:
1925 index.make(apps, sortedids, archapks, repodirs[1], True)
1927 git_remote = config.get('binary_transparency_remote')
1928 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1930 btlog.make_binary_transparency_log(repodirs)
1932 if config['update_stats']:
1933 # Update known apks info...
1934 knownapks.writeifchanged()
1936 # Generate latest apps data for widget
1937 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1939 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1941 appid = line.rstrip()
1942 data += appid + "\t"
1944 data += app.Name + "\t"
1945 if app.icon is not None:
1946 data += app.icon + "\t"
1947 data += app.License + "\n"
1948 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1952 write_cache(apkcache)
1954 # Update the wiki...
1956 update_wiki(apps, sortedids, apks + archapks)
1958 logging.info(_("Finished"))
1961 if __name__ == "__main__":