3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
42 from . import metadata
43 from .common import SdkToolsPopen
44 from .exception import BuildException, FDroidException
48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
49 UNSET_VERSION_CODE = -0x100000000
51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
58 APK_PERMISSION_PAT = \
59 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
62 screen_densities = ['640', '480', '320', '240', '160', '120']
63 screen_resolutions = {
75 all_screen_densities = ['0'] + screen_densities
77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
83 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
86 def dpi_to_px(density):
87 return (int(density) * 48) / 160
91 return (int(px) * 160) / 48
94 def get_icon_dir(repodir, density):
96 return os.path.join(repodir, "icons")
97 return os.path.join(repodir, "icons-%s" % density)
100 def get_icon_dirs(repodir):
101 for density in screen_densities:
102 yield get_icon_dir(repodir, density)
105 def get_all_icon_dirs(repodir):
106 for density in all_screen_densities:
107 yield get_icon_dir(repodir, density)
110 def update_wiki(apps, sortedids, apks):
113 :param apps: fully populated list of all applications
114 :param apks: all apks, except...
116 logging.info("Updating wiki")
118 wikiredircat = 'App Redirects'
120 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
121 path=config['wiki_path'])
122 site.login(config['wiki_user'], config['wiki_password'])
124 generated_redirects = {}
126 for appid in sortedids:
127 app = metadata.App(apps[appid])
131 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
133 for af in app.AntiFeatures:
134 wikidata += '{{AntiFeature|' + af + '}}\n'
139 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
142 app.added.strftime('%Y-%m-%d') if app.added else '',
143 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
158 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
160 wikidata += app.Summary
161 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
163 wikidata += "=Description=\n"
164 wikidata += metadata.description_wiki(app.Description) + "\n"
166 wikidata += "=Maintainer Notes=\n"
167 if app.MaintainerNotes:
168 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
169 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
171 # Get a list of all packages for this application...
173 gotcurrentver = False
177 if apk['packageName'] == appid:
178 if str(apk['versionCode']) == app.CurrentVersionCode:
181 # Include ones we can't build, as a special case...
182 for build in app.builds:
184 if build.versionCode == app.CurrentVersionCode:
186 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
187 apklist.append({'versionCode': int(build.versionCode),
188 'versionName': build.versionName,
189 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
194 if apk['versionCode'] == int(build.versionCode):
199 apklist.append({'versionCode': int(build.versionCode),
200 'versionName': build.versionName,
201 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
203 if app.CurrentVersionCode == '0':
205 # Sort with most recent first...
206 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
208 wikidata += "=Versions=\n"
209 if len(apklist) == 0:
210 wikidata += "We currently have no versions of this app available."
211 elif not gotcurrentver:
212 wikidata += "We don't have the current version of this app."
214 wikidata += "We have the current version of this app."
215 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
216 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
217 if len(app.NoSourceSince) > 0:
218 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
219 if len(app.CurrentVersion) > 0:
220 wikidata += "The current (recommended) version is " + app.CurrentVersion
221 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
224 wikidata += "==" + apk['versionName'] + "==\n"
226 if 'buildproblem' in apk:
227 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
230 wikidata += "This version is built and signed by "
232 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
234 wikidata += "the original developer.\n\n"
235 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
237 wikidata += '\n[[Category:' + wikicat + ']]\n'
238 if len(app.NoSourceSince) > 0:
239 wikidata += '\n[[Category:Apps missing source code]]\n'
240 if validapks == 0 and not app.Disabled:
241 wikidata += '\n[[Category:Apps with no packages]]\n'
242 if cantupdate and not app.Disabled:
243 wikidata += "\n[[Category:Apps we cannot update]]\n"
244 if buildfails and not app.Disabled:
245 wikidata += "\n[[Category:Apps with failing builds]]\n"
246 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
247 wikidata += '\n[[Category:Apps to Update]]\n'
249 wikidata += '\n[[Category:Apps that are disabled]]\n'
250 if app.UpdateCheckMode == 'None' and not app.Disabled:
251 wikidata += '\n[[Category:Apps with no update check]]\n'
252 for appcat in app.Categories:
253 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
255 # We can't have underscores in the page name, even if they're in
256 # the package ID, because MediaWiki messes with them...
257 pagename = appid.replace('_', ' ')
259 # Drop a trailing newline, because mediawiki is going to drop it anyway
260 # and it we don't we'll think the page has changed when it hasn't...
261 if wikidata.endswith('\n'):
262 wikidata = wikidata[:-1]
264 generated_pages[pagename] = wikidata
266 # Make a redirect from the name to the ID too, unless there's
267 # already an existing page with the name and it isn't a redirect.
269 apppagename = app.Name.replace('_', ' ')
270 apppagename = apppagename.replace('{', '')
271 apppagename = apppagename.replace('}', ' ')
272 apppagename = apppagename.replace(':', ' ')
273 apppagename = apppagename.replace('[', ' ')
274 apppagename = apppagename.replace(']', ' ')
275 # Drop double spaces caused mostly by replacing ':' above
276 apppagename = apppagename.replace(' ', ' ')
277 for expagename in site.allpages(prefix=apppagename,
278 filterredir='nonredirects',
280 if expagename == apppagename:
282 # Another reason not to make the redirect page is if the app name
283 # is the same as it's ID, because that will overwrite the real page
284 # with an redirect to itself! (Although it seems like an odd
285 # scenario this happens a lot, e.g. where there is metadata but no
286 # builds or binaries to extract a name from.
287 if apppagename == pagename:
290 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
292 for tcat, genp in [(wikicat, generated_pages),
293 (wikiredircat, generated_redirects)]:
294 catpages = site.Pages['Category:' + tcat]
296 for page in catpages:
297 existingpages.append(page.name)
298 if page.name in genp:
299 pagetxt = page.edit()
300 if pagetxt != genp[page.name]:
301 logging.debug("Updating modified page " + page.name)
302 page.save(genp[page.name], summary='Auto-updated')
304 logging.debug("Page " + page.name + " is unchanged")
306 logging.warn("Deleting page " + page.name)
307 page.delete('No longer published')
308 for pagename, text in genp.items():
309 logging.debug("Checking " + pagename)
310 if pagename not in existingpages:
311 logging.debug("Creating page " + pagename)
313 newpage = site.Pages[pagename]
314 newpage.save(text, summary='Auto-created')
315 except Exception as e:
316 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
318 # Purge server cache to ensure counts are up to date
319 site.pages['Repository Maintenance'].purge()
322 def delete_disabled_builds(apps, apkcache, repodirs):
323 """Delete disabled build outputs.
325 :param apps: list of all applications, as per metadata.read_metadata
326 :param apkcache: current apk cache information
327 :param repodirs: the repo directories to process
329 for appid, app in apps.items():
330 for build in app['builds']:
331 if not build.disable:
333 apkfilename = common.get_release_filename(app, build)
334 iconfilename = "%s.%s.png" % (
337 for repodir in repodirs:
339 os.path.join(repodir, apkfilename),
340 os.path.join(repodir, apkfilename + '.asc'),
341 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
343 for density in all_screen_densities:
344 repo_dir = get_icon_dir(repodir, density)
345 files.append(os.path.join(repo_dir, iconfilename))
348 if os.path.exists(f):
349 logging.info("Deleting disabled build output " + f)
351 if apkfilename in apkcache:
352 del apkcache[apkfilename]
355 def resize_icon(iconpath, density):
357 if not os.path.isfile(iconpath):
362 fp = open(iconpath, 'rb')
364 size = dpi_to_px(density)
366 if any(length > size for length in im.size):
368 im.thumbnail((size, size), Image.ANTIALIAS)
369 logging.debug("%s was too large at %s - new size is %s" % (
370 iconpath, oldsize, im.size))
371 im.save(iconpath, "PNG")
373 except Exception as e:
374 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
381 def resize_all_icons(repodirs):
382 """Resize all icons that exceed the max size
384 :param repodirs: the repo directories to process
386 for repodir in repodirs:
387 for density in screen_densities:
388 icon_dir = get_icon_dir(repodir, density)
389 icon_glob = os.path.join(icon_dir, '*.png')
390 for iconpath in glob.glob(icon_glob):
391 resize_icon(iconpath, density)
395 """ Get the signing certificate of an apk. To get the same md5 has that
396 Android gets, we encode the .RSA certificate in a specific format and pass
397 it hex-encoded to the md5 digest algorithm.
399 :param apkpath: path to the apk
400 :returns: A string containing the md5 of the signature of the apk or None
401 if an error occurred.
404 with zipfile.ZipFile(apkpath, 'r') as apk:
405 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
408 logging.error("Found no signing certificates on %s" % apkpath)
411 logging.error("Found multiple signing certificates on %s" % apkpath)
414 cert = apk.read(certs[0])
416 cert_encoded = common.get_certificate(cert)
418 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
421 def get_cache_file():
422 return os.path.join('tmp', 'apkcache')
427 Gather information about all the apk files in the repo directory,
428 using cached data if possible.
431 apkcachefile = get_cache_file()
432 if not options.clean and os.path.exists(apkcachefile):
433 with open(apkcachefile, 'rb') as cf:
434 apkcache = pickle.load(cf, encoding='utf-8')
435 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
443 def write_cache(apkcache):
444 apkcachefile = get_cache_file()
445 cache_path = os.path.dirname(apkcachefile)
446 if not os.path.exists(cache_path):
447 os.makedirs(cache_path)
448 apkcache["METADATA_VERSION"] = METADATA_VERSION
449 with open(apkcachefile, 'wb') as cf:
450 pickle.dump(apkcache, cf)
453 def get_icon_bytes(apkzip, iconsrc):
454 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
456 return apkzip.read(iconsrc)
458 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
461 def sha256sum(filename):
462 '''Calculate the sha256 of the given file'''
463 sha = hashlib.sha256()
464 with open(filename, 'rb') as f:
470 return sha.hexdigest()
473 def has_known_vulnerability(filename):
474 """checks for known vulnerabilities in the APK
476 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
477 version. Google also enforces this:
478 https://support.google.com/faqs/answer/6376725?hl=en
480 Checks whether there are more than one classes.dex or AndroidManifest.xml
481 files, which is invalid and an essential part of the "Master Key" attack.
483 http://www.saurik.com/id/17
486 # statically load this pattern
487 if not hasattr(has_known_vulnerability, "pattern"):
488 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
491 with zipfile.ZipFile(filename) as zf:
492 for name in zf.namelist():
493 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
496 chunk = lib.read(4096)
499 m = has_known_vulnerability.pattern.search(chunk)
501 version = m.group(1).decode('ascii')
502 if version.startswith('1.0.1') and version[5] >= 'r' \
503 or version.startswith('1.0.2') and version[5] >= 'f':
504 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
506 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
509 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
510 if name in files_in_apk:
512 files_in_apk.add(name)
517 def insert_obbs(repodir, apps, apks):
518 """Scans the .obb files in a given repo directory and adds them to the
519 relevant APK instances. OBB files have versionCodes like APK
520 files, and they are loosely associated. If there is an OBB file
521 present, then any APK with the same or higher versionCode will use
522 that OBB file. There are two OBB types: main and patch, each APK
523 can only have only have one of each.
525 https://developer.android.com/google/play/expansion-files.html
527 :param repodir: repo directory to scan
528 :param apps: list of current, valid apps
529 :param apks: current information on all APKs
533 def obbWarnDelete(f, msg):
534 logging.warning(msg + f)
535 if options.delete_unknown:
536 logging.error("Deleting unknown file: " + f)
540 java_Integer_MIN_VALUE = -pow(2, 31)
541 currentPackageNames = apps.keys()
542 for f in glob.glob(os.path.join(repodir, '*.obb')):
543 obbfile = os.path.basename(f)
544 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
545 chunks = obbfile.split('.')
546 if chunks[0] != 'main' and chunks[0] != 'patch':
547 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
549 if not re.match(r'^-?[0-9]+$', chunks[1]):
550 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
552 versionCode = int(chunks[1])
553 packagename = ".".join(chunks[2:-1])
555 highestVersionCode = java_Integer_MIN_VALUE
556 if packagename not in currentPackageNames:
557 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
560 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
561 highestVersionCode = apk['versionCode']
562 if versionCode > highestVersionCode:
563 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
564 + ') than any APK: ')
566 obbsha256 = sha256sum(f)
567 obbs.append((packagename, versionCode, obbfile, obbsha256))
570 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
571 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
572 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
573 apk['obbMainFile'] = obbfile
574 apk['obbMainFileSha256'] = obbsha256
575 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
576 apk['obbPatchFile'] = obbfile
577 apk['obbPatchFileSha256'] = obbsha256
578 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
582 def _get_localized_dict(app, locale):
583 '''get the dict to add localized store metadata to'''
584 if 'localized' not in app:
585 app['localized'] = collections.OrderedDict()
586 if locale not in app['localized']:
587 app['localized'][locale] = collections.OrderedDict()
588 return app['localized'][locale]
591 def _set_localized_text_entry(app, locale, key, f):
592 limit = config['char_limits'][key]
593 localized = _get_localized_dict(app, locale)
595 text = fp.read()[:limit]
597 localized[key] = text
600 def _set_author_entry(app, key, f):
601 limit = config['char_limits']['author']
603 text = fp.read()[:limit]
608 def copy_triple_t_store_metadata(apps):
609 """Include store metadata from the app's source repo
611 The Triple-T Gradle Play Publisher is a plugin that has a standard
612 file layout for all of the metadata and graphics that the Google
613 Play Store accepts. Since F-Droid has the git repo, it can just
614 pluck those files directly. This method reads any text files into
615 the app dict, then copies any graphics into the fdroid repo
618 This needs to be run before insert_localized_app_metadata() so that
619 the graphics files that are copied into the fdroid repo get
622 https://github.com/Triple-T/gradle-play-publisher#upload-images
623 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
627 if not os.path.isdir('build'):
628 return # nothing to do
630 for packageName, app in apps.items():
631 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
632 logging.debug('Triple-T Gradle Play Publisher: ' + d)
633 for root, dirs, files in os.walk(d):
634 segments = root.split('/')
635 locale = segments[-2]
637 if f == 'fulldescription':
638 _set_localized_text_entry(app, locale, 'description',
639 os.path.join(root, f))
641 elif f == 'shortdescription':
642 _set_localized_text_entry(app, locale, 'summary',
643 os.path.join(root, f))
646 _set_localized_text_entry(app, locale, 'name',
647 os.path.join(root, f))
650 _set_localized_text_entry(app, locale, 'video',
651 os.path.join(root, f))
653 elif f == 'whatsnew':
654 _set_localized_text_entry(app, segments[-1], 'whatsNew',
655 os.path.join(root, f))
657 elif f == 'contactEmail':
658 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
660 elif f == 'contactPhone':
661 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
663 elif f == 'contactWebsite':
664 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
667 base, extension = common.get_extension(f)
668 dirname = os.path.basename(root)
669 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
670 if segments[-2] == 'listing':
671 locale = segments[-3]
673 locale = segments[-2]
674 destdir = os.path.join('repo', packageName, locale)
675 os.makedirs(destdir, mode=0o755, exist_ok=True)
676 sourcefile = os.path.join(root, f)
677 destfile = os.path.join(destdir, dirname + '.' + extension)
678 logging.debug('copying ' + sourcefile + ' ' + destfile)
679 shutil.copy(sourcefile, destfile)
682 def insert_localized_app_metadata(apps):
683 """scans standard locations for graphics and localized text
685 Scans for localized description files, store graphics, and
686 screenshot PNG files in statically defined screenshots directory
687 and adds them to the app metadata. The screenshots and graphic
688 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
689 and must be in the following layout:
690 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
692 repo/packageName/locale/featureGraphic.png
693 repo/packageName/locale/phoneScreenshots/1.png
694 repo/packageName/locale/phoneScreenshots/2.png
696 The changelog files must be text files named with the versionCode
697 ending with ".txt" and must be in the following layout:
698 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
700 repo/packageName/locale/changelogs/12345.txt
702 This will scan the each app's source repo then the metadata/ dir
703 for these standard locations of changelog files. If it finds
704 them, they will be added to the dict of all packages, with the
705 versions in the metadata/ folder taking precendence over the what
706 is in the app's source repo.
708 Where "packageName" is the app's packageName and "locale" is the locale
709 of the graphics, e.g. what language they are in, using the IETF RFC5646
710 format (en-US, fr-CA, es-MX, etc).
712 This will also scan the app's git for a fastlane folder, and the
713 metadata/ folder and the apps' source repos for standard locations
714 of graphic and screenshot files. If it finds them, it will copy
715 them into the repo. The fastlane files follow this pattern:
716 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
720 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
721 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
722 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
724 for srcd in sorted(sourcedirs):
725 if not os.path.isdir(srcd):
727 for root, dirs, files in os.walk(srcd):
728 segments = root.split('/')
729 packageName = segments[1]
730 if packageName not in apps:
731 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
733 locale = segments[-1]
734 destdir = os.path.join('repo', packageName, locale)
736 if f in ('description.txt', 'full_description.txt'):
737 _set_localized_text_entry(apps[packageName], locale, 'description',
738 os.path.join(root, f))
740 elif f in ('summary.txt', 'short_description.txt'):
741 _set_localized_text_entry(apps[packageName], locale, 'summary',
742 os.path.join(root, f))
744 elif f in ('name.txt', 'title.txt'):
745 _set_localized_text_entry(apps[packageName], locale, 'name',
746 os.path.join(root, f))
748 elif f == 'video.txt':
749 _set_localized_text_entry(apps[packageName], locale, 'video',
750 os.path.join(root, f))
752 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
753 locale = segments[-2]
754 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
755 os.path.join(root, f))
758 base, extension = common.get_extension(f)
759 if locale == 'images':
760 locale = segments[-2]
761 destdir = os.path.join('repo', packageName, locale)
762 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
763 os.makedirs(destdir, mode=0o755, exist_ok=True)
764 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
765 shutil.copy(os.path.join(root, f), destdir)
767 if d in SCREENSHOT_DIRS:
768 for f in glob.glob(os.path.join(root, d, '*.*')):
769 _, extension = common.get_extension(f)
770 if extension in ALLOWED_EXTENSIONS:
771 screenshotdestdir = os.path.join(destdir, d)
772 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
773 logging.debug('copying ' + f + ' ' + screenshotdestdir)
774 shutil.copy(f, screenshotdestdir)
776 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
778 if not os.path.isdir(d):
780 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
781 if not os.path.isfile(f):
783 segments = f.split('/')
784 packageName = segments[1]
786 screenshotdir = segments[3]
787 filename = os.path.basename(f)
788 base, extension = common.get_extension(filename)
790 if packageName not in apps:
791 logging.warning('Found "%s" graphic without metadata for app "%s"!'
792 % (filename, packageName))
794 graphics = _get_localized_dict(apps[packageName], locale)
796 if extension not in ALLOWED_EXTENSIONS:
797 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
798 elif base in GRAPHIC_NAMES:
799 # there can only be zero or one of these per locale
800 graphics[base] = filename
801 elif screenshotdir in SCREENSHOT_DIRS:
802 # there can any number of these per locale
803 logging.debug('adding to ' + screenshotdir + ': ' + f)
804 if screenshotdir not in graphics:
805 graphics[screenshotdir] = []
806 graphics[screenshotdir].append(filename)
808 logging.warning('Unsupported graphics file found: ' + f)
811 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
812 """Scan a repo for all files with an extension except APK/OBB
814 :param apkcache: current cached info about all repo files
815 :param repodir: repo directory to scan
816 :param knownapks: list of all known files, as per metadata.read_metadata
817 :param use_date_from_file: use date from file (instead of current date)
818 for newly added files
823 repodir = repodir.encode('utf-8')
824 for name in os.listdir(repodir):
825 file_extension = common.get_file_extension(name)
826 if file_extension == 'apk' or file_extension == 'obb':
828 filename = os.path.join(repodir, name)
829 name_utf8 = name.decode('utf-8')
830 if filename.endswith(b'_src.tar.gz'):
831 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
833 if not common.is_repo_file(filename):
835 stat = os.stat(filename)
836 if stat.st_size == 0:
837 raise FDroidException(filename + ' is zero size!')
839 shasum = sha256sum(filename)
842 repo_file = apkcache[name]
843 # added time is cached as tuple but used here as datetime instance
844 if 'added' in repo_file:
845 a = repo_file['added']
846 if isinstance(a, datetime):
847 repo_file['added'] = a
849 repo_file['added'] = datetime(*a[:6])
850 if repo_file.get('hash') == shasum:
851 logging.debug("Reading " + name_utf8 + " from cache")
854 logging.debug("Ignoring stale cache data for " + name)
857 logging.debug("Processing " + name_utf8)
858 repo_file = collections.OrderedDict()
859 repo_file['name'] = os.path.splitext(name_utf8)[0]
860 # TODO rename apkname globally to something more generic
861 repo_file['apkName'] = name_utf8
862 repo_file['hash'] = shasum
863 repo_file['hashType'] = 'sha256'
864 repo_file['versionCode'] = 0
865 repo_file['versionName'] = shasum
866 # the static ID is the SHA256 unless it is set in the metadata
867 repo_file['packageName'] = shasum
869 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
871 repo_file['packageName'] = m.group(1)
872 repo_file['versionCode'] = int(m.group(2))
873 srcfilename = name + b'_src.tar.gz'
874 if os.path.exists(os.path.join(repodir, srcfilename)):
875 repo_file['srcname'] = srcfilename.decode('utf-8')
876 repo_file['size'] = stat.st_size
878 apkcache[name] = repo_file
881 if use_date_from_file:
882 timestamp = stat.st_ctime
883 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
885 default_date_param = None
887 # Record in knownapks, getting the added date at the same time..
888 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
889 default_date=default_date_param)
891 repo_file['added'] = added
893 repo_files.append(repo_file)
895 return repo_files, cachechanged
898 def scan_apk_aapt(apk, apkfile):
899 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
900 if p.returncode != 0:
901 if options.delete_unknown:
902 if os.path.exists(apkfile):
903 logging.error("Failed to get apk information, deleting " + apkfile)
906 logging.error("Could not find {0} to remove it".format(apkfile))
908 logging.error("Failed to get apk information, skipping " + apkfile)
909 raise BuildException("Invalid APK")
910 for line in p.output.splitlines():
911 if line.startswith("package:"):
913 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
914 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
915 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
916 except Exception as e:
917 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
918 elif line.startswith("application:"):
919 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
920 # Keep path to non-dpi icon in case we need it
921 match = re.match(APK_ICON_PAT_NODPI, line)
923 apk['icons_src']['-1'] = match.group(1)
924 elif line.startswith("launchable-activity:"):
925 # Only use launchable-activity as fallback to application
927 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
928 if '-1' not in apk['icons_src']:
929 match = re.match(APK_ICON_PAT_NODPI, line)
931 apk['icons_src']['-1'] = match.group(1)
932 elif line.startswith("application-icon-"):
933 match = re.match(APK_ICON_PAT, line)
935 density = match.group(1)
936 path = match.group(2)
937 apk['icons_src'][density] = path
938 elif line.startswith("sdkVersion:"):
939 m = re.match(APK_SDK_VERSION_PAT, line)
941 logging.error(line.replace('sdkVersion:', '')
942 + ' is not a valid minSdkVersion!')
944 apk['minSdkVersion'] = m.group(1)
945 # if target not set, default to min
946 if 'targetSdkVersion' not in apk:
947 apk['targetSdkVersion'] = m.group(1)
948 elif line.startswith("targetSdkVersion:"):
949 m = re.match(APK_SDK_VERSION_PAT, line)
951 logging.error(line.replace('targetSdkVersion:', '')
952 + ' is not a valid targetSdkVersion!')
954 apk['targetSdkVersion'] = m.group(1)
955 elif line.startswith("maxSdkVersion:"):
956 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
957 elif line.startswith("native-code:"):
958 apk['nativecode'] = []
959 for arch in line[13:].split(' '):
960 apk['nativecode'].append(arch[1:-1])
961 elif line.startswith('uses-permission:'):
962 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
963 if perm_match['maxSdkVersion']:
964 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
965 permission = UsesPermission(
967 perm_match['maxSdkVersion']
970 apk['uses-permission'].append(permission)
971 elif line.startswith('uses-permission-sdk-23:'):
972 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
973 if perm_match['maxSdkVersion']:
974 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
975 permission_sdk_23 = UsesPermissionSdk23(
977 perm_match['maxSdkVersion']
980 apk['uses-permission-sdk-23'].append(permission_sdk_23)
982 elif line.startswith('uses-feature:'):
983 feature = re.match(APK_FEATURE_PAT, line).group(1)
984 # Filter out this, it's only added with the latest SDK tools and
985 # causes problems for lots of apps.
986 if feature != "android.hardware.screen.portrait" \
987 and feature != "android.hardware.screen.landscape":
988 if feature.startswith("android.feature."):
989 feature = feature[16:]
990 apk['features'].add(feature)
993 def scan_apk_androguard(apk, apkfile):
995 from androguard.core.bytecodes.apk import APK
996 apkobject = APK(apkfile)
997 if apkobject.is_valid_APK():
998 arsc = apkobject.get_android_resources()
1000 if options.delete_unknown:
1001 if os.path.exists(apkfile):
1002 logging.error("Failed to get apk information, deleting " + apkfile)
1005 logging.error("Could not find {0} to remove it".format(apkfile))
1007 logging.error("Failed to get apk information, skipping " + apkfile)
1008 raise BuildException("Invaild APK")
1010 raise FDroidException("androguard library is not installed and aapt not present")
1011 except FileNotFoundError:
1012 logging.error("Could not open apk file for analysis")
1013 raise BuildException("Invalid APK")
1015 apk['packageName'] = apkobject.get_package()
1016 apk['versionCode'] = int(apkobject.get_androidversion_code())
1017 apk['versionName'] = apkobject.get_androidversion_name()
1018 if apk['versionName'][0] == "@":
1019 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1020 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1021 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1022 apk['name'] = apkobject.get_app_name()
1024 if apkobject.get_max_sdk_version() is not None:
1025 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1026 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1027 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1029 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1030 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1032 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1034 for file in apkobject.get_files():
1035 d_re = density_re.match(file)
1037 folder = d_re.group(1).split('-')
1039 resolution = folder[1]
1042 density = screen_resolutions[resolution]
1043 apk['icons_src'][density] = d_re.group(0)
1045 if apk['icons_src'].get('-1') is None:
1046 apk['icons_src']['-1'] = apk['icons_src']['160']
1048 arch_re = re.compile("^lib/(.*)/.*$")
1049 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1051 apk['nativecode'] = []
1052 apk['nativecode'].extend(sorted(list(arch)))
1054 xml = apkobject.get_android_manifest_xml()
1056 for item in xml.getElementsByTagName('uses-permission'):
1057 name = str(item.getAttribute("android:name"))
1058 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1059 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1060 permission = UsesPermission(
1064 apk['uses-permission'].append(permission)
1066 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1067 name = str(item.getAttribute("android:name"))
1068 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1069 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1070 permission_sdk_23 = UsesPermissionSdk23(
1074 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1076 for item in xml.getElementsByTagName('uses-feature'):
1077 feature = str(item.getAttribute("android:name"))
1078 if feature != "android.hardware.screen.portrait" \
1079 and feature != "android.hardware.screen.landscape":
1080 if feature.startswith("android.feature."):
1081 feature = feature[16:]
1082 apk['features'].append(feature)
1085 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1086 allow_disabled_algorithms=False, archive_bad_sig=False):
1087 """Scan the apk with the given filename in the given repo directory.
1089 This also extracts the icons.
1091 :param apkcache: current apk cache information
1092 :param apkfilename: the filename of the apk to scan
1093 :param repodir: repo directory to scan
1094 :param knownapks: known apks info
1095 :param use_date_from_apk: use date from APK (instead of current date)
1096 for newly added APKs
1097 :param allow_disabled_algorithms: allow APKs with valid signatures that include
1098 disabled algorithms in the signature (e.g. MD5)
1099 :param archive_bad_sig: move APKs with a bad signature to the archive
1100 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1101 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1104 if ' ' in apkfilename:
1105 if options.rename_apks:
1106 newfilename = apkfilename.replace(' ', '_')
1107 os.rename(os.path.join(repodir, apkfilename),
1108 os.path.join(repodir, newfilename))
1109 apkfilename = newfilename
1111 logging.critical("Spaces in filenames are not allowed.")
1112 return True, None, False
1114 apkfile = os.path.join(repodir, apkfilename)
1115 shasum = sha256sum(apkfile)
1117 cachechanged = False
1119 if apkfilename in apkcache:
1120 apk = apkcache[apkfilename]
1121 if apk.get('hash') == shasum:
1122 logging.debug("Reading " + apkfilename + " from cache")
1125 logging.debug("Ignoring stale cache data for " + apkfilename)
1128 logging.debug("Processing " + apkfilename)
1130 apk['hash'] = shasum
1131 apk['hashType'] = 'sha256'
1132 apk['uses-permission'] = []
1133 apk['uses-permission-sdk-23'] = []
1134 apk['features'] = []
1135 apk['icons_src'] = {}
1137 apk['antiFeatures'] = set()
1140 if SdkToolsPopen(['aapt', 'version'], output=False):
1141 scan_apk_aapt(apk, apkfile)
1143 scan_apk_androguard(apk, apkfile)
1144 except BuildException:
1145 return True, None, False
1147 if 'minSdkVersion' not in apk:
1148 logging.warn("No SDK version information found in {0}".format(apkfile))
1149 apk['minSdkVersion'] = 1
1151 # Check for debuggable apks...
1152 if common.isApkAndDebuggable(apkfile):
1153 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1155 # Get the signature (or md5 of, to be precise)...
1156 logging.debug('Getting signature of {0}'.format(apkfile))
1157 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1159 logging.critical("Failed to get apk signature")
1160 return True, None, False
1162 if options.rename_apks:
1163 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1164 std_short_name = os.path.join(repodir, n)
1165 if apkfile != std_short_name:
1166 if os.path.exists(std_short_name):
1167 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1168 if apkfile != std_long_name:
1169 if os.path.exists(std_long_name):
1170 dupdir = os.path.join('duplicates', repodir)
1171 if not os.path.isdir(dupdir):
1172 os.makedirs(dupdir, exist_ok=True)
1173 dupfile = os.path.join('duplicates', std_long_name)
1174 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1175 os.rename(apkfile, dupfile)
1176 return True, None, False
1178 os.rename(apkfile, std_long_name)
1179 apkfile = std_long_name
1181 os.rename(apkfile, std_short_name)
1182 apkfile = std_short_name
1183 apkfilename = apkfile[len(repodir) + 1:]
1185 apk['apkName'] = apkfilename
1186 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1187 if os.path.exists(os.path.join(repodir, srcfilename)):
1188 apk['srcname'] = srcfilename
1189 apk['size'] = os.path.getsize(apkfile)
1191 # verify the jar signature is correct, allow deprecated
1192 # algorithms only if the APK is in the archive.
1194 if not common.verify_apk_signature(apkfile):
1195 if repodir == 'archive' or allow_disabled_algorithms:
1196 if common.verify_old_apk_signature(apkfile):
1197 apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1205 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1206 move_apk_between_sections(repodir, 'archive', apk)
1208 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1209 return True, None, False
1211 if 'KnownVuln' not in apk['antiFeatures']:
1212 if has_known_vulnerability(apkfile):
1213 apk['antiFeatures'].add('KnownVuln')
1215 apkzip = zipfile.ZipFile(apkfile, 'r')
1217 # if an APK has files newer than the system time, suggest updating
1218 # the system clock. This is useful for offline systems, used for
1219 # signing, which do not have another source of clock sync info. It
1220 # has to be more than 24 hours newer because ZIP/APK files do not
1221 # store timezone info
1222 manifest = apkzip.getinfo('AndroidManifest.xml')
1223 if manifest.date_time[1] == 0: # month can't be zero
1224 logging.debug('AndroidManifest.xml has no date')
1226 dt_obj = datetime(*manifest.date_time)
1227 checkdt = dt_obj - timedelta(1)
1228 if datetime.today() < checkdt:
1229 logging.warn('System clock is older than manifest in: '
1231 + '\nSet clock to that time using:\n'
1232 + 'sudo date -s "' + str(dt_obj) + '"')
1234 iconfilename = "%s.%s.png" % (
1238 # Extract the icon file...
1239 empty_densities = []
1240 for density in screen_densities:
1241 if density not in apk['icons_src']:
1242 empty_densities.append(density)
1244 iconsrc = apk['icons_src'][density]
1245 icon_dir = get_icon_dir(repodir, density)
1246 icondest = os.path.join(icon_dir, iconfilename)
1249 with open(icondest, 'wb') as f:
1250 f.write(get_icon_bytes(apkzip, iconsrc))
1251 apk['icons'][density] = iconfilename
1252 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1253 logging.warning("Error retrieving icon file: %s" % (icondest))
1254 del apk['icons_src'][density]
1255 empty_densities.append(density)
1257 if '-1' in apk['icons_src']:
1258 iconsrc = apk['icons_src']['-1']
1259 iconpath = os.path.join(
1260 get_icon_dir(repodir, '0'), iconfilename)
1261 with open(iconpath, 'wb') as f:
1262 f.write(get_icon_bytes(apkzip, iconsrc))
1264 im = Image.open(iconpath)
1265 dpi = px_to_dpi(im.size[0])
1266 for density in screen_densities:
1267 if density in apk['icons']:
1269 if density == screen_densities[-1] or dpi >= int(density):
1270 apk['icons'][density] = iconfilename
1271 shutil.move(iconpath,
1272 os.path.join(get_icon_dir(repodir, density), iconfilename))
1273 empty_densities.remove(density)
1275 except Exception as e:
1276 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1279 apk['icon'] = iconfilename
1283 # First try resizing down to not lose quality
1285 for density in screen_densities:
1286 if density not in empty_densities:
1287 last_density = density
1289 if last_density is None:
1291 logging.debug("Density %s not available, resizing down from %s"
1292 % (density, last_density))
1294 last_iconpath = os.path.join(
1295 get_icon_dir(repodir, last_density), iconfilename)
1296 iconpath = os.path.join(
1297 get_icon_dir(repodir, density), iconfilename)
1300 fp = open(last_iconpath, 'rb')
1303 size = dpi_to_px(density)
1305 im.thumbnail((size, size), Image.ANTIALIAS)
1306 im.save(iconpath, "PNG")
1307 empty_densities.remove(density)
1308 except Exception as e:
1309 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1314 # Then just copy from the highest resolution available
1316 for density in reversed(screen_densities):
1317 if density not in empty_densities:
1318 last_density = density
1320 if last_density is None:
1322 logging.debug("Density %s not available, copying from lower density %s"
1323 % (density, last_density))
1326 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1327 os.path.join(get_icon_dir(repodir, density), iconfilename))
1329 empty_densities.remove(density)
1331 for density in screen_densities:
1332 icon_dir = get_icon_dir(repodir, density)
1333 icondest = os.path.join(icon_dir, iconfilename)
1334 resize_icon(icondest, density)
1336 # Copy from icons-mdpi to icons since mdpi is the baseline density
1337 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1338 if os.path.isfile(baseline):
1339 apk['icons']['0'] = iconfilename
1340 shutil.copyfile(baseline,
1341 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1343 if use_date_from_apk and manifest.date_time[1] != 0:
1344 default_date_param = datetime(*manifest.date_time)
1346 default_date_param = None
1348 # Record in known apks, getting the added date at the same time..
1349 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1350 default_date=default_date_param)
1352 apk['added'] = added
1354 apkcache[apkfilename] = apk
1357 return False, apk, cachechanged
1360 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1361 """Scan the apks in the given repo directory.
1363 This also extracts the icons.
1365 :param apkcache: current apk cache information
1366 :param repodir: repo directory to scan
1367 :param knownapks: known apks info
1368 :param use_date_from_apk: use date from APK (instead of current date)
1369 for newly added APKs
1370 :returns: (apks, cachechanged) where apks is a list of apk information,
1371 and cachechanged is True if the apkcache got changed.
1374 cachechanged = False
1376 for icon_dir in get_all_icon_dirs(repodir):
1377 if os.path.exists(icon_dir):
1379 shutil.rmtree(icon_dir)
1380 os.makedirs(icon_dir)
1382 os.makedirs(icon_dir)
1385 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1386 apkfilename = apkfile[len(repodir) + 1:]
1387 ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1388 (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks,
1389 use_date_from_apk, ada, True)
1394 return apks, cachechanged
1397 def apply_info_from_latest_apk(apps, apks):
1399 Some information from the apks needs to be applied up to the application level.
1400 When doing this, we use the info from the most recent version's apk.
1401 We deal with figuring out when the app was added and last updated at the same time.
1403 for appid, app in apps.items():
1404 bestver = UNSET_VERSION_CODE
1406 if apk['packageName'] == appid:
1407 if apk['versionCode'] > bestver:
1408 bestver = apk['versionCode']
1412 if not app.added or apk['added'] < app.added:
1413 app.added = apk['added']
1414 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1415 app.lastUpdated = apk['added']
1418 logging.debug("Don't know when " + appid + " was added")
1419 if not app.lastUpdated:
1420 logging.debug("Don't know when " + appid + " was last updated")
1422 if bestver == UNSET_VERSION_CODE:
1424 if app.Name is None:
1425 app.Name = app.AutoName or appid
1427 logging.debug("Application " + appid + " has no packages")
1429 if app.Name is None:
1430 app.Name = bestapk['name']
1431 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1432 if app.CurrentVersionCode is None:
1433 app.CurrentVersionCode = str(bestver)
1436 def make_categories_txt(repodir, categories):
1437 '''Write a category list in the repo to allow quick access'''
1439 for cat in sorted(categories):
1440 catdata += cat + '\n'
1441 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1445 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1447 def filter_apk_list_sorted(apk_list):
1449 for apk in apk_list:
1450 if apk['packageName'] == appid:
1453 # Sort the apk list by version code. First is highest/newest.
1454 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1456 for appid, app in apps.items():
1458 if app.ArchivePolicy:
1459 keepversions = int(app.ArchivePolicy[:-9])
1461 keepversions = defaultkeepversions
1463 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1464 .format(appid, len(apks), keepversions, len(archapks)))
1466 current_app_apks = filter_apk_list_sorted(apks)
1467 if len(current_app_apks) > keepversions:
1468 # Move back the ones we don't want.
1469 for apk in current_app_apks[keepversions:]:
1470 move_apk_between_sections(repodir, archivedir, apk)
1471 archapks.append(apk)
1474 current_app_archapks = filter_apk_list_sorted(archapks)
1475 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1477 # Move forward the ones we want again, except DisableAlgorithm
1478 for apk in current_app_archapks:
1479 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1480 move_apk_between_sections(archivedir, repodir, apk)
1481 archapks.remove(apk)
1484 if kept == keepversions:
1488 def move_apk_between_sections(from_dir, to_dir, apk):
1489 """move an APK from repo to archive or vice versa"""
1491 def _move_file(from_dir, to_dir, filename, ignore_missing):
1492 from_path = os.path.join(from_dir, filename)
1493 if ignore_missing and not os.path.exists(from_path):
1495 to_path = os.path.join(to_dir, filename)
1496 if not os.path.exists(to_dir):
1498 shutil.move(from_path, to_path)
1500 if from_dir == to_dir:
1503 logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1504 _move_file(from_dir, to_dir, apk['apkName'], False)
1505 _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1506 for density in all_screen_densities:
1507 from_icon_dir = get_icon_dir(from_dir, density)
1508 to_icon_dir = get_icon_dir(to_dir, density)
1509 if density not in apk['icons']:
1511 _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1512 if 'srcname' in apk:
1513 _move_file(from_dir, to_dir, apk['srcname'], False)
1516 def add_apks_to_per_app_repos(repodir, apks):
1517 apks_per_app = dict()
1519 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1520 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1521 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1522 apks_per_app[apk['packageName']] = apk
1524 if not os.path.exists(apk['per_app_icons']):
1525 logging.info('Adding new repo for only ' + apk['packageName'])
1526 os.makedirs(apk['per_app_icons'])
1528 apkpath = os.path.join(repodir, apk['apkName'])
1529 shutil.copy(apkpath, apk['per_app_repo'])
1530 apksigpath = apkpath + '.sig'
1531 if os.path.exists(apksigpath):
1532 shutil.copy(apksigpath, apk['per_app_repo'])
1533 apkascpath = apkpath + '.asc'
1534 if os.path.exists(apkascpath):
1535 shutil.copy(apkascpath, apk['per_app_repo'])
1544 global config, options
1546 # Parse command line...
1547 parser = ArgumentParser()
1548 common.setup_global_opts(parser)
1549 parser.add_argument("--create-key", action="store_true", default=False,
1550 help="Create a repo signing key in a keystore")
1551 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1552 help="Create skeleton metadata files that are missing")
1553 parser.add_argument("--delete-unknown", action="store_true", default=False,
1554 help="Delete APKs and/or OBBs without metadata from the repo")
1555 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1556 help="Report on build data status")
1557 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1558 help="Interactively ask about things that need updating.")
1559 parser.add_argument("-I", "--icons", action="store_true", default=False,
1560 help="Resize all the icons exceeding the max pixel size and exit")
1561 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1562 help="Specify editor to use in interactive mode. Default " +
1563 "is /etc/alternatives/editor")
1564 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1565 help="Update the wiki")
1566 parser.add_argument("--pretty", action="store_true", default=False,
1567 help="Produce human-readable index.xml")
1568 parser.add_argument("--clean", action="store_true", default=False,
1569 help="Clean update - don't uses caches, reprocess all apks")
1570 parser.add_argument("--nosign", action="store_true", default=False,
1571 help="When configured for signed indexes, create only unsigned indexes at this stage")
1572 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1573 help="Use date from apk instead of current time for newly added apks")
1574 parser.add_argument("--rename-apks", action="store_true", default=False,
1575 help="Rename APK files that do not match package.name_123.apk")
1576 parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1577 help="Include APKs that are signed with disabled algorithms like MD5")
1578 metadata.add_metadata_arguments(parser)
1579 options = parser.parse_args()
1580 metadata.warnings_action = options.W
1582 config = common.read_config(options)
1584 if not ('jarsigner' in config and 'keytool' in config):
1585 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1588 if config['archive_older'] != 0:
1589 repodirs.append('archive')
1590 if not os.path.exists('archive'):
1594 resize_all_icons(repodirs)
1597 if options.rename_apks:
1598 options.clean = True
1600 # check that icons exist now, rather than fail at the end of `fdroid update`
1601 for k in ['repo_icon', 'archive_icon']:
1603 if not os.path.exists(config[k]):
1604 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1607 # if the user asks to create a keystore, do it now, reusing whatever it can
1608 if options.create_key:
1609 if os.path.exists(config['keystore']):
1610 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1611 logging.critical("\t'" + config['keystore'] + "'")
1614 if 'repo_keyalias' not in config:
1615 config['repo_keyalias'] = socket.getfqdn()
1616 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1617 if 'keydname' not in config:
1618 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1619 common.write_to_config(config, 'keydname', config['keydname'])
1620 if 'keystore' not in config:
1621 config['keystore'] = common.default_config['keystore']
1622 common.write_to_config(config, 'keystore', config['keystore'])
1624 password = common.genpassword()
1625 if 'keystorepass' not in config:
1626 config['keystorepass'] = password
1627 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1628 if 'keypass' not in config:
1629 config['keypass'] = password
1630 common.write_to_config(config, 'keypass', config['keypass'])
1631 common.genkeystore(config)
1634 apps = metadata.read_metadata()
1636 # Generate a list of categories...
1638 for app in apps.values():
1639 categories.update(app.Categories)
1641 # Read known apks data (will be updated and written back when we've finished)
1642 knownapks = common.KnownApks()
1645 apkcache = get_cache()
1647 # Delete builds for disabled apps
1648 delete_disabled_builds(apps, apkcache, repodirs)
1650 # Scan all apks in the main repo
1651 apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1653 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1654 options.use_date_from_apk)
1655 cachechanged = cachechanged or fcachechanged
1657 # Generate warnings for apk's with no metadata (or create skeleton
1658 # metadata files, if requested on the command line)
1661 if apk['packageName'] not in apps:
1662 if options.create_metadata:
1663 if 'name' not in apk:
1664 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1666 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1667 f.write("License:Unknown\n")
1668 f.write("Web Site:\n")
1669 f.write("Source Code:\n")
1670 f.write("Issue Tracker:\n")
1671 f.write("Changelog:\n")
1672 f.write("Summary:" + apk['name'] + "\n")
1673 f.write("Description:\n")
1674 f.write(apk['name'] + "\n")
1676 f.write("Name:" + apk['name'] + "\n")
1678 logging.info("Generated skeleton metadata for " + apk['packageName'])
1681 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1682 if options.delete_unknown:
1683 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1684 rmf = os.path.join(repodirs[0], apk['apkName'])
1685 if not os.path.exists(rmf):
1686 logging.error("Could not find {0} to remove it".format(rmf))
1690 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1692 # update the metadata with the newly created ones included
1694 apps = metadata.read_metadata()
1696 copy_triple_t_store_metadata(apps)
1697 insert_obbs(repodirs[0], apps, apks)
1698 insert_localized_app_metadata(apps)
1700 # Scan the archive repo for apks as well
1701 if len(repodirs) > 1:
1702 archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1708 # Apply information from latest apks to the application and update dates
1709 apply_info_from_latest_apk(apps, apks + archapks)
1711 # Sort the app list by name, then the web site doesn't have to by default.
1712 # (we had to wait until we'd scanned the apks to do this, because mostly the
1713 # name comes from there!)
1714 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1716 # APKs are placed into multiple repos based on the app package, providing
1717 # per-app subscription feeds for nightly builds and things like it
1718 if config['per_app_repos']:
1719 add_apks_to_per_app_repos(repodirs[0], apks)
1720 for appid, app in apps.items():
1721 repodir = os.path.join(appid, 'fdroid', 'repo')
1723 appdict[appid] = app
1724 if os.path.isdir(repodir):
1725 index.make(appdict, [appid], apks, repodir, False)
1727 logging.info('Skipping index generation for ' + appid)
1730 if len(repodirs) > 1:
1731 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1733 # Make the index for the main repo...
1734 index.make(apps, sortedids, apks, repodirs[0], False)
1735 make_categories_txt(repodirs[0], categories)
1737 # If there's an archive repo, make the index for it. We already scanned it
1739 if len(repodirs) > 1:
1740 index.make(apps, sortedids, archapks, repodirs[1], True)
1742 git_remote = config.get('binary_transparency_remote')
1743 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1745 btlog.make_binary_transparency_log(repodirs)
1747 if config['update_stats']:
1748 # Update known apks info...
1749 knownapks.writeifchanged()
1751 # Generate latest apps data for widget
1752 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1754 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1756 appid = line.rstrip()
1757 data += appid + "\t"
1759 data += app.Name + "\t"
1760 if app.icon is not None:
1761 data += app.icon + "\t"
1762 data += app.License + "\n"
1763 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1767 write_cache(apkcache)
1769 # Update the wiki...
1771 update_wiki(apps, sortedids, apks + archapks)
1773 logging.info("Finished.")
1776 if __name__ == "__main__":