3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
42 from . import metadata
43 from .common import SdkToolsPopen
44 from .exception import BuildException, FDroidException
48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
49 UNSET_VERSION_CODE = -0x100000000
51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
58 APK_PERMISSION_PAT = \
59 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
62 screen_densities = ['640', '480', '320', '240', '160', '120']
63 screen_resolutions = {
75 all_screen_densities = ['0'] + screen_densities
77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
83 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
86 def dpi_to_px(density):
87 return (int(density) * 48) / 160
91 return (int(px) * 160) / 48
94 def get_icon_dir(repodir, density):
96 return os.path.join(repodir, "icons")
97 return os.path.join(repodir, "icons-%s" % density)
100 def get_icon_dirs(repodir):
101 for density in screen_densities:
102 yield get_icon_dir(repodir, density)
105 def get_all_icon_dirs(repodir):
106 for density in all_screen_densities:
107 yield get_icon_dir(repodir, density)
110 def update_wiki(apps, sortedids, apks):
113 :param apps: fully populated list of all applications
114 :param apks: all apks, except...
116 logging.info("Updating wiki")
118 wikiredircat = 'App Redirects'
120 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
121 path=config['wiki_path'])
122 site.login(config['wiki_user'], config['wiki_password'])
124 generated_redirects = {}
126 for appid in sortedids:
127 app = metadata.App(apps[appid])
131 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
133 for af in app.AntiFeatures:
134 wikidata += '{{AntiFeature|' + af + '}}\n'
139 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
142 app.added.strftime('%Y-%m-%d') if app.added else '',
143 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
158 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
160 wikidata += app.Summary
161 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
163 wikidata += "=Description=\n"
164 wikidata += metadata.description_wiki(app.Description) + "\n"
166 wikidata += "=Maintainer Notes=\n"
167 if app.MaintainerNotes:
168 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
169 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
171 # Get a list of all packages for this application...
173 gotcurrentver = False
177 if apk['packageName'] == appid:
178 if str(apk['versionCode']) == app.CurrentVersionCode:
181 # Include ones we can't build, as a special case...
182 for build in app.builds:
184 if build.versionCode == app.CurrentVersionCode:
186 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
187 apklist.append({'versionCode': int(build.versionCode),
188 'versionName': build.versionName,
189 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
194 if apk['versionCode'] == int(build.versionCode):
199 apklist.append({'versionCode': int(build.versionCode),
200 'versionName': build.versionName,
201 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
203 if app.CurrentVersionCode == '0':
205 # Sort with most recent first...
206 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
208 wikidata += "=Versions=\n"
209 if len(apklist) == 0:
210 wikidata += "We currently have no versions of this app available."
211 elif not gotcurrentver:
212 wikidata += "We don't have the current version of this app."
214 wikidata += "We have the current version of this app."
215 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
216 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
217 if len(app.NoSourceSince) > 0:
218 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
219 if len(app.CurrentVersion) > 0:
220 wikidata += "The current (recommended) version is " + app.CurrentVersion
221 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
224 wikidata += "==" + apk['versionName'] + "==\n"
226 if 'buildproblem' in apk:
227 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
230 wikidata += "This version is built and signed by "
232 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
234 wikidata += "the original developer.\n\n"
235 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
237 wikidata += '\n[[Category:' + wikicat + ']]\n'
238 if len(app.NoSourceSince) > 0:
239 wikidata += '\n[[Category:Apps missing source code]]\n'
240 if validapks == 0 and not app.Disabled:
241 wikidata += '\n[[Category:Apps with no packages]]\n'
242 if cantupdate and not app.Disabled:
243 wikidata += "\n[[Category:Apps we cannot update]]\n"
244 if buildfails and not app.Disabled:
245 wikidata += "\n[[Category:Apps with failing builds]]\n"
246 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
247 wikidata += '\n[[Category:Apps to Update]]\n'
249 wikidata += '\n[[Category:Apps that are disabled]]\n'
250 if app.UpdateCheckMode == 'None' and not app.Disabled:
251 wikidata += '\n[[Category:Apps with no update check]]\n'
252 for appcat in app.Categories:
253 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
255 # We can't have underscores in the page name, even if they're in
256 # the package ID, because MediaWiki messes with them...
257 pagename = appid.replace('_', ' ')
259 # Drop a trailing newline, because mediawiki is going to drop it anyway
260 # and it we don't we'll think the page has changed when it hasn't...
261 if wikidata.endswith('\n'):
262 wikidata = wikidata[:-1]
264 generated_pages[pagename] = wikidata
266 # Make a redirect from the name to the ID too, unless there's
267 # already an existing page with the name and it isn't a redirect.
269 apppagename = app.Name.replace('_', ' ')
270 apppagename = apppagename.replace('{', '')
271 apppagename = apppagename.replace('}', ' ')
272 apppagename = apppagename.replace(':', ' ')
273 apppagename = apppagename.replace('[', ' ')
274 apppagename = apppagename.replace(']', ' ')
275 # Drop double spaces caused mostly by replacing ':' above
276 apppagename = apppagename.replace(' ', ' ')
277 for expagename in site.allpages(prefix=apppagename,
278 filterredir='nonredirects',
280 if expagename == apppagename:
282 # Another reason not to make the redirect page is if the app name
283 # is the same as it's ID, because that will overwrite the real page
284 # with an redirect to itself! (Although it seems like an odd
285 # scenario this happens a lot, e.g. where there is metadata but no
286 # builds or binaries to extract a name from.
287 if apppagename == pagename:
290 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
292 for tcat, genp in [(wikicat, generated_pages),
293 (wikiredircat, generated_redirects)]:
294 catpages = site.Pages['Category:' + tcat]
296 for page in catpages:
297 existingpages.append(page.name)
298 if page.name in genp:
299 pagetxt = page.edit()
300 if pagetxt != genp[page.name]:
301 logging.debug("Updating modified page " + page.name)
302 page.save(genp[page.name], summary='Auto-updated')
304 logging.debug("Page " + page.name + " is unchanged")
306 logging.warn("Deleting page " + page.name)
307 page.delete('No longer published')
308 for pagename, text in genp.items():
309 logging.debug("Checking " + pagename)
310 if pagename not in existingpages:
311 logging.debug("Creating page " + pagename)
313 newpage = site.Pages[pagename]
314 newpage.save(text, summary='Auto-created')
315 except Exception as e:
316 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
318 # Purge server cache to ensure counts are up to date
319 site.pages['Repository Maintenance'].purge()
322 def delete_disabled_builds(apps, apkcache, repodirs):
323 """Delete disabled build outputs.
325 :param apps: list of all applications, as per metadata.read_metadata
326 :param apkcache: current apk cache information
327 :param repodirs: the repo directories to process
329 for appid, app in apps.items():
330 for build in app['builds']:
331 if not build.disable:
333 apkfilename = common.get_release_filename(app, build)
334 iconfilename = "%s.%s.png" % (
337 for repodir in repodirs:
339 os.path.join(repodir, apkfilename),
340 os.path.join(repodir, apkfilename + '.asc'),
341 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
343 for density in all_screen_densities:
344 repo_dir = get_icon_dir(repodir, density)
345 files.append(os.path.join(repo_dir, iconfilename))
348 if os.path.exists(f):
349 logging.info("Deleting disabled build output " + f)
351 if apkfilename in apkcache:
352 del apkcache[apkfilename]
355 def resize_icon(iconpath, density):
357 if not os.path.isfile(iconpath):
362 fp = open(iconpath, 'rb')
364 size = dpi_to_px(density)
366 if any(length > size for length in im.size):
368 im.thumbnail((size, size), Image.ANTIALIAS)
369 logging.debug("%s was too large at %s - new size is %s" % (
370 iconpath, oldsize, im.size))
371 im.save(iconpath, "PNG")
373 except Exception as e:
374 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
381 def resize_all_icons(repodirs):
382 """Resize all icons that exceed the max size
384 :param repodirs: the repo directories to process
386 for repodir in repodirs:
387 for density in screen_densities:
388 icon_dir = get_icon_dir(repodir, density)
389 icon_glob = os.path.join(icon_dir, '*.png')
390 for iconpath in glob.glob(icon_glob):
391 resize_icon(iconpath, density)
395 """ Get the signing certificate of an apk. To get the same md5 has that
396 Android gets, we encode the .RSA certificate in a specific format and pass
397 it hex-encoded to the md5 digest algorithm.
399 :param apkpath: path to the apk
400 :returns: A string containing the md5 of the signature of the apk or None
401 if an error occurred.
404 with zipfile.ZipFile(apkpath, 'r') as apk:
405 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
408 logging.error("Found no signing certificates on %s" % apkpath)
411 logging.error("Found multiple signing certificates on %s" % apkpath)
414 cert = apk.read(certs[0])
416 cert_encoded = common.get_certificate(cert)
418 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
421 def get_cache_file():
422 return os.path.join('tmp', 'apkcache')
426 """Get the cached dict of the APK index
428 Gather information about all the apk files in the repo directory,
429 using cached data if possible. Some of the index operations take a
430 long time, like calculating the SHA-256 and verifying the APK
433 The cache is invalidated if the metadata version is different, or
434 the 'allow_disabled_algorithms' config/option is different. In
435 those cases, there is no easy way to know what has changed from
436 the cache, so just rerun the whole thing.
441 apkcachefile = get_cache_file()
442 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
443 if not options.clean and os.path.exists(apkcachefile):
444 with open(apkcachefile, 'rb') as cf:
445 apkcache = pickle.load(cf, encoding='utf-8')
446 if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
447 or apkcache.get('allow_disabled_algorithms') != ada:
452 apkcache["METADATA_VERSION"] = METADATA_VERSION
453 apkcache['allow_disabled_algorithms'] = ada
458 def write_cache(apkcache):
459 apkcachefile = get_cache_file()
460 cache_path = os.path.dirname(apkcachefile)
461 if not os.path.exists(cache_path):
462 os.makedirs(cache_path)
463 with open(apkcachefile, 'wb') as cf:
464 pickle.dump(apkcache, cf)
467 def get_icon_bytes(apkzip, iconsrc):
468 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
470 return apkzip.read(iconsrc)
472 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
475 def sha256sum(filename):
476 '''Calculate the sha256 of the given file'''
477 sha = hashlib.sha256()
478 with open(filename, 'rb') as f:
484 return sha.hexdigest()
487 def has_known_vulnerability(filename):
488 """checks for known vulnerabilities in the APK
490 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
491 version. Google also enforces this:
492 https://support.google.com/faqs/answer/6376725?hl=en
494 Checks whether there are more than one classes.dex or AndroidManifest.xml
495 files, which is invalid and an essential part of the "Master Key" attack.
497 http://www.saurik.com/id/17
500 # statically load this pattern
501 if not hasattr(has_known_vulnerability, "pattern"):
502 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
505 with zipfile.ZipFile(filename) as zf:
506 for name in zf.namelist():
507 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
510 chunk = lib.read(4096)
513 m = has_known_vulnerability.pattern.search(chunk)
515 version = m.group(1).decode('ascii')
516 if version.startswith('1.0.1') and version[5] >= 'r' \
517 or version.startswith('1.0.2') and version[5] >= 'f':
518 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
520 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
523 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
524 if name in files_in_apk:
526 files_in_apk.add(name)
531 def insert_obbs(repodir, apps, apks):
532 """Scans the .obb files in a given repo directory and adds them to the
533 relevant APK instances. OBB files have versionCodes like APK
534 files, and they are loosely associated. If there is an OBB file
535 present, then any APK with the same or higher versionCode will use
536 that OBB file. There are two OBB types: main and patch, each APK
537 can only have only have one of each.
539 https://developer.android.com/google/play/expansion-files.html
541 :param repodir: repo directory to scan
542 :param apps: list of current, valid apps
543 :param apks: current information on all APKs
547 def obbWarnDelete(f, msg):
548 logging.warning(msg + f)
549 if options.delete_unknown:
550 logging.error("Deleting unknown file: " + f)
554 java_Integer_MIN_VALUE = -pow(2, 31)
555 currentPackageNames = apps.keys()
556 for f in glob.glob(os.path.join(repodir, '*.obb')):
557 obbfile = os.path.basename(f)
558 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
559 chunks = obbfile.split('.')
560 if chunks[0] != 'main' and chunks[0] != 'patch':
561 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
563 if not re.match(r'^-?[0-9]+$', chunks[1]):
564 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
566 versionCode = int(chunks[1])
567 packagename = ".".join(chunks[2:-1])
569 highestVersionCode = java_Integer_MIN_VALUE
570 if packagename not in currentPackageNames:
571 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
574 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
575 highestVersionCode = apk['versionCode']
576 if versionCode > highestVersionCode:
577 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
578 + ') than any APK: ')
580 obbsha256 = sha256sum(f)
581 obbs.append((packagename, versionCode, obbfile, obbsha256))
584 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
585 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
586 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
587 apk['obbMainFile'] = obbfile
588 apk['obbMainFileSha256'] = obbsha256
589 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
590 apk['obbPatchFile'] = obbfile
591 apk['obbPatchFileSha256'] = obbsha256
592 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
596 def _get_localized_dict(app, locale):
597 '''get the dict to add localized store metadata to'''
598 if 'localized' not in app:
599 app['localized'] = collections.OrderedDict()
600 if locale not in app['localized']:
601 app['localized'][locale] = collections.OrderedDict()
602 return app['localized'][locale]
605 def _set_localized_text_entry(app, locale, key, f):
606 limit = config['char_limits'][key]
607 localized = _get_localized_dict(app, locale)
609 text = fp.read()[:limit]
611 localized[key] = text
614 def _set_author_entry(app, key, f):
615 limit = config['char_limits']['author']
617 text = fp.read()[:limit]
622 def copy_triple_t_store_metadata(apps):
623 """Include store metadata from the app's source repo
625 The Triple-T Gradle Play Publisher is a plugin that has a standard
626 file layout for all of the metadata and graphics that the Google
627 Play Store accepts. Since F-Droid has the git repo, it can just
628 pluck those files directly. This method reads any text files into
629 the app dict, then copies any graphics into the fdroid repo
632 This needs to be run before insert_localized_app_metadata() so that
633 the graphics files that are copied into the fdroid repo get
636 https://github.com/Triple-T/gradle-play-publisher#upload-images
637 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
641 if not os.path.isdir('build'):
642 return # nothing to do
644 for packageName, app in apps.items():
645 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
646 logging.debug('Triple-T Gradle Play Publisher: ' + d)
647 for root, dirs, files in os.walk(d):
648 segments = root.split('/')
649 locale = segments[-2]
651 if f == 'fulldescription':
652 _set_localized_text_entry(app, locale, 'description',
653 os.path.join(root, f))
655 elif f == 'shortdescription':
656 _set_localized_text_entry(app, locale, 'summary',
657 os.path.join(root, f))
660 _set_localized_text_entry(app, locale, 'name',
661 os.path.join(root, f))
664 _set_localized_text_entry(app, locale, 'video',
665 os.path.join(root, f))
667 elif f == 'whatsnew':
668 _set_localized_text_entry(app, segments[-1], 'whatsNew',
669 os.path.join(root, f))
671 elif f == 'contactEmail':
672 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
674 elif f == 'contactPhone':
675 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
677 elif f == 'contactWebsite':
678 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
681 base, extension = common.get_extension(f)
682 dirname = os.path.basename(root)
683 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
684 if segments[-2] == 'listing':
685 locale = segments[-3]
687 locale = segments[-2]
688 destdir = os.path.join('repo', packageName, locale)
689 os.makedirs(destdir, mode=0o755, exist_ok=True)
690 sourcefile = os.path.join(root, f)
691 destfile = os.path.join(destdir, dirname + '.' + extension)
692 logging.debug('copying ' + sourcefile + ' ' + destfile)
693 shutil.copy(sourcefile, destfile)
696 def insert_localized_app_metadata(apps):
697 """scans standard locations for graphics and localized text
699 Scans for localized description files, store graphics, and
700 screenshot PNG files in statically defined screenshots directory
701 and adds them to the app metadata. The screenshots and graphic
702 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
703 and must be in the following layout:
704 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
706 repo/packageName/locale/featureGraphic.png
707 repo/packageName/locale/phoneScreenshots/1.png
708 repo/packageName/locale/phoneScreenshots/2.png
710 The changelog files must be text files named with the versionCode
711 ending with ".txt" and must be in the following layout:
712 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
714 repo/packageName/locale/changelogs/12345.txt
716 This will scan the each app's source repo then the metadata/ dir
717 for these standard locations of changelog files. If it finds
718 them, they will be added to the dict of all packages, with the
719 versions in the metadata/ folder taking precendence over the what
720 is in the app's source repo.
722 Where "packageName" is the app's packageName and "locale" is the locale
723 of the graphics, e.g. what language they are in, using the IETF RFC5646
724 format (en-US, fr-CA, es-MX, etc).
726 This will also scan the app's git for a fastlane folder, and the
727 metadata/ folder and the apps' source repos for standard locations
728 of graphic and screenshot files. If it finds them, it will copy
729 them into the repo. The fastlane files follow this pattern:
730 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
734 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
735 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
736 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
738 for srcd in sorted(sourcedirs):
739 if not os.path.isdir(srcd):
741 for root, dirs, files in os.walk(srcd):
742 segments = root.split('/')
743 packageName = segments[1]
744 if packageName not in apps:
745 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
747 locale = segments[-1]
748 destdir = os.path.join('repo', packageName, locale)
750 if f in ('description.txt', 'full_description.txt'):
751 _set_localized_text_entry(apps[packageName], locale, 'description',
752 os.path.join(root, f))
754 elif f in ('summary.txt', 'short_description.txt'):
755 _set_localized_text_entry(apps[packageName], locale, 'summary',
756 os.path.join(root, f))
758 elif f in ('name.txt', 'title.txt'):
759 _set_localized_text_entry(apps[packageName], locale, 'name',
760 os.path.join(root, f))
762 elif f == 'video.txt':
763 _set_localized_text_entry(apps[packageName], locale, 'video',
764 os.path.join(root, f))
766 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
767 locale = segments[-2]
768 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
769 os.path.join(root, f))
772 base, extension = common.get_extension(f)
773 if locale == 'images':
774 locale = segments[-2]
775 destdir = os.path.join('repo', packageName, locale)
776 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
777 os.makedirs(destdir, mode=0o755, exist_ok=True)
778 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
779 shutil.copy(os.path.join(root, f), destdir)
781 if d in SCREENSHOT_DIRS:
782 for f in glob.glob(os.path.join(root, d, '*.*')):
783 _, extension = common.get_extension(f)
784 if extension in ALLOWED_EXTENSIONS:
785 screenshotdestdir = os.path.join(destdir, d)
786 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
787 logging.debug('copying ' + f + ' ' + screenshotdestdir)
788 shutil.copy(f, screenshotdestdir)
790 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
792 if not os.path.isdir(d):
794 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
795 if not os.path.isfile(f):
797 segments = f.split('/')
798 packageName = segments[1]
800 screenshotdir = segments[3]
801 filename = os.path.basename(f)
802 base, extension = common.get_extension(filename)
804 if packageName not in apps:
805 logging.warning('Found "%s" graphic without metadata for app "%s"!'
806 % (filename, packageName))
808 graphics = _get_localized_dict(apps[packageName], locale)
810 if extension not in ALLOWED_EXTENSIONS:
811 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
812 elif base in GRAPHIC_NAMES:
813 # there can only be zero or one of these per locale
814 graphics[base] = filename
815 elif screenshotdir in SCREENSHOT_DIRS:
816 # there can any number of these per locale
817 logging.debug('adding to ' + screenshotdir + ': ' + f)
818 if screenshotdir not in graphics:
819 graphics[screenshotdir] = []
820 graphics[screenshotdir].append(filename)
822 logging.warning('Unsupported graphics file found: ' + f)
825 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
826 """Scan a repo for all files with an extension except APK/OBB
828 :param apkcache: current cached info about all repo files
829 :param repodir: repo directory to scan
830 :param knownapks: list of all known files, as per metadata.read_metadata
831 :param use_date_from_file: use date from file (instead of current date)
832 for newly added files
837 repodir = repodir.encode('utf-8')
838 for name in os.listdir(repodir):
839 file_extension = common.get_file_extension(name)
840 if file_extension == 'apk' or file_extension == 'obb':
842 filename = os.path.join(repodir, name)
843 name_utf8 = name.decode('utf-8')
844 if filename.endswith(b'_src.tar.gz'):
845 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
847 if not common.is_repo_file(filename):
849 stat = os.stat(filename)
850 if stat.st_size == 0:
851 raise FDroidException(filename + ' is zero size!')
853 shasum = sha256sum(filename)
856 repo_file = apkcache[name]
857 # added time is cached as tuple but used here as datetime instance
858 if 'added' in repo_file:
859 a = repo_file['added']
860 if isinstance(a, datetime):
861 repo_file['added'] = a
863 repo_file['added'] = datetime(*a[:6])
864 if repo_file.get('hash') == shasum:
865 logging.debug("Reading " + name_utf8 + " from cache")
868 logging.debug("Ignoring stale cache data for " + name)
871 logging.debug("Processing " + name_utf8)
872 repo_file = collections.OrderedDict()
873 repo_file['name'] = os.path.splitext(name_utf8)[0]
874 # TODO rename apkname globally to something more generic
875 repo_file['apkName'] = name_utf8
876 repo_file['hash'] = shasum
877 repo_file['hashType'] = 'sha256'
878 repo_file['versionCode'] = 0
879 repo_file['versionName'] = shasum
880 # the static ID is the SHA256 unless it is set in the metadata
881 repo_file['packageName'] = shasum
883 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
885 repo_file['packageName'] = m.group(1)
886 repo_file['versionCode'] = int(m.group(2))
887 srcfilename = name + b'_src.tar.gz'
888 if os.path.exists(os.path.join(repodir, srcfilename)):
889 repo_file['srcname'] = srcfilename.decode('utf-8')
890 repo_file['size'] = stat.st_size
892 apkcache[name] = repo_file
895 if use_date_from_file:
896 timestamp = stat.st_ctime
897 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
899 default_date_param = None
901 # Record in knownapks, getting the added date at the same time..
902 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
903 default_date=default_date_param)
905 repo_file['added'] = added
907 repo_files.append(repo_file)
909 return repo_files, cachechanged
912 def scan_apk(apk_file):
914 Scans an APK file and returns dictionary with metadata of the APK.
916 Attention: This does *not* verify that the APK signature is correct.
918 :param apk_file: The (ideally absolute) path to the APK file
919 :raises BuildException
920 :return A dict containing APK metadata
923 'hash': sha256sum(apk_file),
924 'hashType': 'sha256',
925 'uses-permission': [],
926 'uses-permission-sdk-23': [],
930 'antiFeatures': set(),
933 if SdkToolsPopen(['aapt', 'version'], output=False):
934 scan_apk_aapt(apk, apk_file)
936 scan_apk_androguard(apk, apk_file)
939 logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
940 apk['sig'] = getsig(apk_file)
942 raise BuildException("Failed to get apk signature")
944 # Get size of the APK
945 apk['size'] = os.path.getsize(apk_file)
947 if 'minSdkVersion' not in apk:
948 logging.warning("No SDK version information found in {0}".format(apk_file))
949 apk['minSdkVersion'] = 1
951 # Check for known vulnerabilities
952 if has_known_vulnerability(apk_file):
953 apk['antiFeatures'].add('KnownVuln')
958 def scan_apk_aapt(apk, apkfile):
959 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
960 if p.returncode != 0:
961 if options.delete_unknown:
962 if os.path.exists(apkfile):
963 logging.error("Failed to get apk information, deleting " + apkfile)
966 logging.error("Could not find {0} to remove it".format(apkfile))
968 logging.error("Failed to get apk information, skipping " + apkfile)
969 raise BuildException("Invalid APK")
970 for line in p.output.splitlines():
971 if line.startswith("package:"):
973 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
974 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
975 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
976 except Exception as e:
977 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
978 elif line.startswith("application:"):
979 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
980 # Keep path to non-dpi icon in case we need it
981 match = re.match(APK_ICON_PAT_NODPI, line)
983 apk['icons_src']['-1'] = match.group(1)
984 elif line.startswith("launchable-activity:"):
985 # Only use launchable-activity as fallback to application
987 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
988 if '-1' not in apk['icons_src']:
989 match = re.match(APK_ICON_PAT_NODPI, line)
991 apk['icons_src']['-1'] = match.group(1)
992 elif line.startswith("application-icon-"):
993 match = re.match(APK_ICON_PAT, line)
995 density = match.group(1)
996 path = match.group(2)
997 apk['icons_src'][density] = path
998 elif line.startswith("sdkVersion:"):
999 m = re.match(APK_SDK_VERSION_PAT, line)
1001 logging.error(line.replace('sdkVersion:', '')
1002 + ' is not a valid minSdkVersion!')
1004 apk['minSdkVersion'] = m.group(1)
1005 # if target not set, default to min
1006 if 'targetSdkVersion' not in apk:
1007 apk['targetSdkVersion'] = m.group(1)
1008 elif line.startswith("targetSdkVersion:"):
1009 m = re.match(APK_SDK_VERSION_PAT, line)
1011 logging.error(line.replace('targetSdkVersion:', '')
1012 + ' is not a valid targetSdkVersion!')
1014 apk['targetSdkVersion'] = m.group(1)
1015 elif line.startswith("maxSdkVersion:"):
1016 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1017 elif line.startswith("native-code:"):
1018 apk['nativecode'] = []
1019 for arch in line[13:].split(' '):
1020 apk['nativecode'].append(arch[1:-1])
1021 elif line.startswith('uses-permission:'):
1022 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1023 if perm_match['maxSdkVersion']:
1024 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1025 permission = UsesPermission(
1027 perm_match['maxSdkVersion']
1030 apk['uses-permission'].append(permission)
1031 elif line.startswith('uses-permission-sdk-23:'):
1032 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1033 if perm_match['maxSdkVersion']:
1034 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1035 permission_sdk_23 = UsesPermissionSdk23(
1037 perm_match['maxSdkVersion']
1040 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1042 elif line.startswith('uses-feature:'):
1043 feature = re.match(APK_FEATURE_PAT, line).group(1)
1044 # Filter out this, it's only added with the latest SDK tools and
1045 # causes problems for lots of apps.
1046 if feature != "android.hardware.screen.portrait" \
1047 and feature != "android.hardware.screen.landscape":
1048 if feature.startswith("android.feature."):
1049 feature = feature[16:]
1050 apk['features'].add(feature)
1053 def scan_apk_androguard(apk, apkfile):
1055 from androguard.core.bytecodes.apk import APK
1056 apkobject = APK(apkfile)
1057 if apkobject.is_valid_APK():
1058 arsc = apkobject.get_android_resources()
1060 if options.delete_unknown:
1061 if os.path.exists(apkfile):
1062 logging.error("Failed to get apk information, deleting " + apkfile)
1065 logging.error("Could not find {0} to remove it".format(apkfile))
1067 logging.error("Failed to get apk information, skipping " + apkfile)
1068 raise BuildException("Invaild APK")
1070 raise FDroidException("androguard library is not installed and aapt not present")
1071 except FileNotFoundError:
1072 logging.error("Could not open apk file for analysis")
1073 raise BuildException("Invalid APK")
1075 apk['packageName'] = apkobject.get_package()
1076 apk['versionCode'] = int(apkobject.get_androidversion_code())
1077 apk['versionName'] = apkobject.get_androidversion_name()
1078 if apk['versionName'][0] == "@":
1079 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1080 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1081 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1082 apk['name'] = apkobject.get_app_name()
1084 if apkobject.get_max_sdk_version() is not None:
1085 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1086 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1087 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1089 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1090 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1092 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1094 for file in apkobject.get_files():
1095 d_re = density_re.match(file)
1097 folder = d_re.group(1).split('-')
1099 resolution = folder[1]
1102 density = screen_resolutions[resolution]
1103 apk['icons_src'][density] = d_re.group(0)
1105 if apk['icons_src'].get('-1') is None:
1106 apk['icons_src']['-1'] = apk['icons_src']['160']
1108 arch_re = re.compile("^lib/(.*)/.*$")
1109 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1111 apk['nativecode'] = []
1112 apk['nativecode'].extend(sorted(list(arch)))
1114 xml = apkobject.get_android_manifest_xml()
1116 for item in xml.getElementsByTagName('uses-permission'):
1117 name = str(item.getAttribute("android:name"))
1118 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1119 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1120 permission = UsesPermission(
1124 apk['uses-permission'].append(permission)
1126 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1127 name = str(item.getAttribute("android:name"))
1128 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1129 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1130 permission_sdk_23 = UsesPermissionSdk23(
1134 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1136 for item in xml.getElementsByTagName('uses-feature'):
1137 feature = str(item.getAttribute("android:name"))
1138 if feature != "android.hardware.screen.portrait" \
1139 and feature != "android.hardware.screen.landscape":
1140 if feature.startswith("android.feature."):
1141 feature = feature[16:]
1142 apk['features'].append(feature)
1145 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1146 allow_disabled_algorithms=False, archive_bad_sig=False):
1147 """Processes the apk with the given filename in the given repo directory.
1149 This also extracts the icons.
1151 :param apkcache: current apk cache information
1152 :param apkfilename: the filename of the apk to scan
1153 :param repodir: repo directory to scan
1154 :param knownapks: known apks info
1155 :param use_date_from_apk: use date from APK (instead of current date)
1156 for newly added APKs
1157 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1158 disabled algorithms in the signature (e.g. MD5)
1159 :param archive_bad_sig: move APKs with a bad signature to the archive
1160 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1161 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1164 if ' ' in apkfilename:
1165 if options.rename_apks:
1166 newfilename = apkfilename.replace(' ', '_')
1167 os.rename(os.path.join(repodir, apkfilename),
1168 os.path.join(repodir, newfilename))
1169 apkfilename = newfilename
1171 logging.critical("Spaces in filenames are not allowed.")
1172 return True, None, False
1175 apkfile = os.path.join(repodir, apkfilename)
1177 cachechanged = False
1179 if apkfilename in apkcache:
1180 apk = apkcache[apkfilename]
1181 if apk.get('hash') == sha256sum(apkfile):
1182 logging.debug("Reading " + apkfilename + " from cache")
1185 logging.debug("Ignoring stale cache data for " + apkfilename)
1188 logging.debug("Processing " + apkfilename)
1191 apk = scan_apk(apkfile)
1192 except BuildException:
1193 logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1194 return True, None, False
1196 # Check for debuggable apks...
1197 if common.isApkAndDebuggable(apkfile):
1198 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1200 if options.rename_apks:
1201 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1202 std_short_name = os.path.join(repodir, n)
1203 if apkfile != std_short_name:
1204 if os.path.exists(std_short_name):
1205 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1206 if apkfile != std_long_name:
1207 if os.path.exists(std_long_name):
1208 dupdir = os.path.join('duplicates', repodir)
1209 if not os.path.isdir(dupdir):
1210 os.makedirs(dupdir, exist_ok=True)
1211 dupfile = os.path.join('duplicates', std_long_name)
1212 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1213 os.rename(apkfile, dupfile)
1214 return True, None, False
1216 os.rename(apkfile, std_long_name)
1217 apkfile = std_long_name
1219 os.rename(apkfile, std_short_name)
1220 apkfile = std_short_name
1221 apkfilename = apkfile[len(repodir) + 1:]
1223 apk['apkName'] = apkfilename
1224 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1225 if os.path.exists(os.path.join(repodir, srcfilename)):
1226 apk['srcname'] = srcfilename
1228 # verify the jar signature is correct, allow deprecated
1229 # algorithms only if the APK is in the archive.
1231 if not common.verify_apk_signature(apkfile):
1232 if repodir == 'archive' or allow_disabled_algorithms:
1233 if common.verify_old_apk_signature(apkfile):
1234 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1242 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1243 move_apk_between_sections(repodir, 'archive', apk)
1245 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1246 return True, None, False
1248 apkzip = zipfile.ZipFile(apkfile, 'r')
1250 # if an APK has files newer than the system time, suggest updating
1251 # the system clock. This is useful for offline systems, used for
1252 # signing, which do not have another source of clock sync info. It
1253 # has to be more than 24 hours newer because ZIP/APK files do not
1254 # store timezone info
1255 manifest = apkzip.getinfo('AndroidManifest.xml')
1256 if manifest.date_time[1] == 0: # month can't be zero
1257 logging.debug('AndroidManifest.xml has no date')
1259 dt_obj = datetime(*manifest.date_time)
1260 checkdt = dt_obj - timedelta(1)
1261 if datetime.today() < checkdt:
1262 logging.warning('System clock is older than manifest in: '
1264 + '\nSet clock to that time using:\n'
1265 + 'sudo date -s "' + str(dt_obj) + '"')
1267 # extract icons from APK zip file
1268 iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1270 empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1272 apkzip.close() # ensure that APK zip file gets closed
1274 # resize existing icons for densities missing in the APK
1275 fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1277 if use_date_from_apk and manifest.date_time[1] != 0:
1278 default_date_param = datetime(*manifest.date_time)
1280 default_date_param = None
1282 # Record in known apks, getting the added date at the same time..
1283 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1284 default_date=default_date_param)
1286 apk['added'] = added
1288 apkcache[apkfilename] = apk
1291 return False, apk, cachechanged
1294 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1295 """Processes the apks in the given repo directory.
1297 This also extracts the icons.
1299 :param apkcache: current apk cache information
1300 :param repodir: repo directory to scan
1301 :param knownapks: known apks info
1302 :param use_date_from_apk: use date from APK (instead of current date)
1303 for newly added APKs
1304 :returns: (apks, cachechanged) where apks is a list of apk information,
1305 and cachechanged is True if the apkcache got changed.
1308 cachechanged = False
1310 for icon_dir in get_all_icon_dirs(repodir):
1311 if os.path.exists(icon_dir):
1313 shutil.rmtree(icon_dir)
1314 os.makedirs(icon_dir)
1316 os.makedirs(icon_dir)
1319 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1320 apkfilename = apkfile[len(repodir) + 1:]
1321 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1322 (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1323 use_date_from_apk, ada, True)
1327 cachechanged = cachechanged or cachethis
1329 return apks, cachechanged
1332 def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir):
1334 Extracts icons from the given APK zip in various densities,
1335 saves them into given repo directory
1336 and stores their names in the APK metadata dictionary.
1338 :param icon_filename: A string representing the icon's file name
1339 :param apk: A populated dictionary containing APK metadata.
1340 Needs to have 'icons_src' key
1341 :param apk_zip: An opened zipfile.ZipFile of the APK file
1342 :param repo_dir: The directory of the APK's repository
1343 :return: A list of icon densities that are missing
1345 empty_densities = []
1346 for density in screen_densities:
1347 if density not in apk['icons_src']:
1348 empty_densities.append(density)
1350 icon_src = apk['icons_src'][density]
1351 icon_dir = get_icon_dir(repo_dir, density)
1352 icon_dest = os.path.join(icon_dir, icon_filename)
1354 # Extract the icon files per density
1356 with open(icon_dest, 'wb') as f:
1357 f.write(get_icon_bytes(apk_zip, icon_src))
1358 apk['icons'][density] = icon_filename
1359 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1360 logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1361 del apk['icons_src'][density]
1362 empty_densities.append(density)
1364 if '-1' in apk['icons_src']:
1365 icon_src = apk['icons_src']['-1']
1366 icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1367 with open(icon_path, 'wb') as f:
1368 f.write(get_icon_bytes(apk_zip, icon_src))
1370 im = Image.open(icon_path)
1371 dpi = px_to_dpi(im.size[0])
1372 for density in screen_densities:
1373 if density in apk['icons']:
1375 if density == screen_densities[-1] or dpi >= int(density):
1376 apk['icons'][density] = icon_filename
1377 shutil.move(icon_path,
1378 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1379 empty_densities.remove(density)
1381 except Exception as e:
1382 logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1385 apk['icon'] = icon_filename
1387 return empty_densities
1390 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1392 Resize existing icons for densities missing in the APK to ensure all densities are available
1394 :param empty_densities: A list of icon densities that are missing
1395 :param icon_filename: A string representing the icon's file name
1396 :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1397 :param repo_dir: The directory of the APK's repository
1399 # First try resizing down to not lose quality
1401 for density in screen_densities:
1402 if density not in empty_densities:
1403 last_density = density
1405 if last_density is None:
1407 logging.debug("Density %s not available, resizing down from %s", density, last_density)
1409 last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1410 icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1413 fp = open(last_icon_path, 'rb')
1416 size = dpi_to_px(density)
1418 im.thumbnail((size, size), Image.ANTIALIAS)
1419 im.save(icon_path, "PNG")
1420 empty_densities.remove(density)
1421 except Exception as e:
1422 logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1427 # Then just copy from the highest resolution available
1429 for density in reversed(screen_densities):
1430 if density not in empty_densities:
1431 last_density = density
1434 if last_density is None:
1438 os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1439 os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1441 empty_densities.remove(density)
1443 for density in screen_densities:
1444 icon_dir = get_icon_dir(repo_dir, density)
1445 icon_dest = os.path.join(icon_dir, icon_filename)
1446 resize_icon(icon_dest, density)
1448 # Copy from icons-mdpi to icons since mdpi is the baseline density
1449 baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1450 if os.path.isfile(baseline):
1451 apk['icons']['0'] = icon_filename
1452 shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1455 def apply_info_from_latest_apk(apps, apks):
1457 Some information from the apks needs to be applied up to the application level.
1458 When doing this, we use the info from the most recent version's apk.
1459 We deal with figuring out when the app was added and last updated at the same time.
1461 for appid, app in apps.items():
1462 bestver = UNSET_VERSION_CODE
1464 if apk['packageName'] == appid:
1465 if apk['versionCode'] > bestver:
1466 bestver = apk['versionCode']
1470 if not app.added or apk['added'] < app.added:
1471 app.added = apk['added']
1472 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1473 app.lastUpdated = apk['added']
1476 logging.debug("Don't know when " + appid + " was added")
1477 if not app.lastUpdated:
1478 logging.debug("Don't know when " + appid + " was last updated")
1480 if bestver == UNSET_VERSION_CODE:
1482 if app.Name is None:
1483 app.Name = app.AutoName or appid
1485 logging.debug("Application " + appid + " has no packages")
1487 if app.Name is None:
1488 app.Name = bestapk['name']
1489 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1490 if app.CurrentVersionCode is None:
1491 app.CurrentVersionCode = str(bestver)
1494 def make_categories_txt(repodir, categories):
1495 '''Write a category list in the repo to allow quick access'''
1497 for cat in sorted(categories):
1498 catdata += cat + '\n'
1499 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1503 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1505 def filter_apk_list_sorted(apk_list):
1507 for apk in apk_list:
1508 if apk['packageName'] == appid:
1511 # Sort the apk list by version code. First is highest/newest.
1512 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1514 for appid, app in apps.items():
1516 if app.ArchivePolicy:
1517 keepversions = int(app.ArchivePolicy[:-9])
1519 keepversions = defaultkeepversions
1521 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1522 .format(appid, len(apks), keepversions, len(archapks)))
1524 current_app_apks = filter_apk_list_sorted(apks)
1525 if len(current_app_apks) > keepversions:
1526 # Move back the ones we don't want.
1527 for apk in current_app_apks[keepversions:]:
1528 move_apk_between_sections(repodir, archivedir, apk)
1529 archapks.append(apk)
1532 current_app_archapks = filter_apk_list_sorted(archapks)
1533 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1535 # Move forward the ones we want again, except DisableAlgorithm
1536 for apk in current_app_archapks:
1537 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1538 move_apk_between_sections(archivedir, repodir, apk)
1539 archapks.remove(apk)
1542 if kept == keepversions:
1546 def move_apk_between_sections(from_dir, to_dir, apk):
1547 """move an APK from repo to archive or vice versa"""
1549 def _move_file(from_dir, to_dir, filename, ignore_missing):
1550 from_path = os.path.join(from_dir, filename)
1551 if ignore_missing and not os.path.exists(from_path):
1553 to_path = os.path.join(to_dir, filename)
1554 if not os.path.exists(to_dir):
1556 shutil.move(from_path, to_path)
1558 if from_dir == to_dir:
1561 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1562 _move_file(from_dir, to_dir, apk['apkName'], False)
1563 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1564 for density in all_screen_densities:
1565 from_icon_dir = get_icon_dir(from_dir, density)
1566 to_icon_dir = get_icon_dir(to_dir, density)
1567 if density not in apk['icons']:
1569 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1570 if 'srcname' in apk:
1571 _move_file(from_dir, to_dir, apk['srcname'], False)
1574 def add_apks_to_per_app_repos(repodir, apks):
1575 apks_per_app = dict()
1577 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1578 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1579 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1580 apks_per_app[apk['packageName']] = apk
1582 if not os.path.exists(apk['per_app_icons']):
1583 logging.info('Adding new repo for only ' + apk['packageName'])
1584 os.makedirs(apk['per_app_icons'])
1586 apkpath = os.path.join(repodir, apk['apkName'])
1587 shutil.copy(apkpath, apk['per_app_repo'])
1588 apksigpath = apkpath + '.sig'
1589 if os.path.exists(apksigpath):
1590 shutil.copy(apksigpath, apk['per_app_repo'])
1591 apkascpath = apkpath + '.asc'
1592 if os.path.exists(apkascpath):
1593 shutil.copy(apkascpath, apk['per_app_repo'])
1602 global config, options
1604 # Parse command line...
1605 parser = ArgumentParser()
1606 common.setup_global_opts(parser)
1607 parser.add_argument("--create-key", action="store_true", default=False,
1608 help="Create a repo signing key in a keystore")
1609 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1610 help="Create skeleton metadata files that are missing")
1611 parser.add_argument("--delete-unknown", action="store_true", default=False,
1612 help="Delete APKs and/or OBBs without metadata from the repo")
1613 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1614 help="Report on build data status")
1615 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1616 help="Interactively ask about things that need updating.")
1617 parser.add_argument("-I", "--icons", action="store_true", default=False,
1618 help="Resize all the icons exceeding the max pixel size and exit")
1619 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1620 help="Specify editor to use in interactive mode. Default " +
1621 "is /etc/alternatives/editor")
1622 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1623 help="Update the wiki")
1624 parser.add_argument("--pretty", action="store_true", default=False,
1625 help="Produce human-readable index.xml")
1626 parser.add_argument("--clean", action="store_true", default=False,
1627 help="Clean update - don't uses caches, reprocess all apks")
1628 parser.add_argument("--nosign", action="store_true", default=False,
1629 help="When configured for signed indexes, create only unsigned indexes at this stage")
1630 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1631 help="Use date from apk instead of current time for newly added apks")
1632 parser.add_argument("--rename-apks", action="store_true", default=False,
1633 help="Rename APK files that do not match package.name_123.apk")
1634 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1635 help="Include APKs that are signed with disabled algorithms like MD5")
1636 metadata.add_metadata_arguments(parser)
1637 options = parser.parse_args()
1638 metadata.warnings_action = options.W
1640 config = common.read_config(options)
1642 if not ('jarsigner' in config and 'keytool' in config):
1643 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1646 if config['archive_older'] != 0:
1647 repodirs.append('archive')
1648 if not os.path.exists('archive'):
1652 resize_all_icons(repodirs)
1655 if options.rename_apks:
1656 options.clean = True
1658 # check that icons exist now, rather than fail at the end of `fdroid update`
1659 for k in ['repo_icon', 'archive_icon']:
1661 if not os.path.exists(config[k]):
1662 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1665 # if the user asks to create a keystore, do it now, reusing whatever it can
1666 if options.create_key:
1667 if os.path.exists(config['keystore']):
1668 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1669 logging.critical("\t'" + config['keystore'] + "'")
1672 if 'repo_keyalias' not in config:
1673 config['repo_keyalias'] = socket.getfqdn()
1674 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1675 if 'keydname' not in config:
1676 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1677 common.write_to_config(config, 'keydname', config['keydname'])
1678 if 'keystore' not in config:
1679 config['keystore'] = common.default_config['keystore']
1680 common.write_to_config(config, 'keystore', config['keystore'])
1682 password = common.genpassword()
1683 if 'keystorepass' not in config:
1684 config['keystorepass'] = password
1685 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1686 if 'keypass' not in config:
1687 config['keypass'] = password
1688 common.write_to_config(config, 'keypass', config['keypass'])
1689 common.genkeystore(config)
1692 apps = metadata.read_metadata()
1694 # Generate a list of categories...
1696 for app in apps.values():
1697 categories.update(app.Categories)
1699 # Read known apks data (will be updated and written back when we've finished)
1700 knownapks = common.KnownApks()
1703 apkcache = get_cache()
1705 # Delete builds for disabled apps
1706 delete_disabled_builds(apps, apkcache, repodirs)
1708 # Scan all apks in the main repo
1709 apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1711 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1712 options.use_date_from_apk)
1713 cachechanged = cachechanged or fcachechanged
1715 # Generate warnings for apk's with no metadata (or create skeleton
1716 # metadata files, if requested on the command line)
1719 if apk['packageName'] not in apps:
1720 if options.create_metadata:
1721 if 'name' not in apk:
1722 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1724 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1725 f.write("License:Unknown\n")
1726 f.write("Web Site:\n")
1727 f.write("Source Code:\n")
1728 f.write("Issue Tracker:\n")
1729 f.write("Changelog:\n")
1730 f.write("Summary:" + apk['name'] + "\n")
1731 f.write("Description:\n")
1732 f.write(apk['name'] + "\n")
1734 f.write("Name:" + apk['name'] + "\n")
1736 logging.info("Generated skeleton metadata for " + apk['packageName'])
1739 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1740 if options.delete_unknown:
1741 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1742 rmf = os.path.join(repodirs[0], apk['apkName'])
1743 if not os.path.exists(rmf):
1744 logging.error("Could not find {0} to remove it".format(rmf))
1748 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1750 # update the metadata with the newly created ones included
1752 apps = metadata.read_metadata()
1754 copy_triple_t_store_metadata(apps)
1755 insert_obbs(repodirs[0], apps, apks)
1756 insert_localized_app_metadata(apps)
1758 # Scan the archive repo for apks as well
1759 if len(repodirs) > 1:
1760 archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1766 # Apply information from latest apks to the application and update dates
1767 apply_info_from_latest_apk(apps, apks + archapks)
1769 # Sort the app list by name, then the web site doesn't have to by default.
1770 # (we had to wait until we'd scanned the apks to do this, because mostly the
1771 # name comes from there!)
1772 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1774 # APKs are placed into multiple repos based on the app package, providing
1775 # per-app subscription feeds for nightly builds and things like it
1776 if config['per_app_repos']:
1777 add_apks_to_per_app_repos(repodirs[0], apks)
1778 for appid, app in apps.items():
1779 repodir = os.path.join(appid, 'fdroid', 'repo')
1781 appdict[appid] = app
1782 if os.path.isdir(repodir):
1783 index.make(appdict, [appid], apks, repodir, False)
1785 logging.info('Skipping index generation for ' + appid)
1788 if len(repodirs) > 1:
1789 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1791 # Make the index for the main repo...
1792 index.make(apps, sortedids, apks, repodirs[0], False)
1793 make_categories_txt(repodirs[0], categories)
1795 # If there's an archive repo, make the index for it. We already scanned it
1797 if len(repodirs) > 1:
1798 index.make(apps, sortedids, archapks, repodirs[1], True)
1800 git_remote = config.get('binary_transparency_remote')
1801 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1803 btlog.make_binary_transparency_log(repodirs)
1805 if config['update_stats']:
1806 # Update known apks info...
1807 knownapks.writeifchanged()
1809 # Generate latest apps data for widget
1810 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1812 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1814 appid = line.rstrip()
1815 data += appid + "\t"
1817 data += app.Name + "\t"
1818 if app.icon is not None:
1819 data += app.icon + "\t"
1820 data += app.License + "\n"
1821 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1825 write_cache(apkcache)
1827 # Update the wiki...
1829 update_wiki(apps, sortedids, apks + archapks)
1831 logging.info("Finished.")
1834 if __name__ == "__main__":