3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
76 all_screen_densities = ['0'] + screen_densities
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 def dpi_to_px(density):
88 return (int(density) * 48) / 160
92 return (int(px) * 160) / 48
95 def get_icon_dir(repodir, density):
97 return os.path.join(repodir, "icons")
98 return os.path.join(repodir, "icons-%s" % density)
101 def get_icon_dirs(repodir):
102 for density in screen_densities:
103 yield get_icon_dir(repodir, density)
106 def get_all_icon_dirs(repodir):
107 for density in all_screen_densities:
108 yield get_icon_dir(repodir, density)
111 def update_wiki(apps, sortedids, apks):
114 :param apps: fully populated list of all applications
115 :param apks: all apks, except...
117 logging.info("Updating wiki")
119 wikiredircat = 'App Redirects'
121 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122 path=config['wiki_path'])
123 site.login(config['wiki_user'], config['wiki_password'])
125 generated_redirects = {}
127 for appid in sortedids:
128 app = metadata.App(apps[appid])
132 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
134 for af in app.AntiFeatures:
135 wikidata += '{{AntiFeature|' + af + '}}\n'
140 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
143 app.added.strftime('%Y-%m-%d') if app.added else '',
144 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
159 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
161 wikidata += app.Summary
162 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
164 wikidata += "=Description=\n"
165 wikidata += metadata.description_wiki(app.Description) + "\n"
167 wikidata += "=Maintainer Notes=\n"
168 if app.MaintainerNotes:
169 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
172 # Get a list of all packages for this application...
174 gotcurrentver = False
178 if apk['packageName'] == appid:
179 if str(apk['versionCode']) == app.CurrentVersionCode:
182 # Include ones we can't build, as a special case...
183 for build in app.builds:
185 if build.versionCode == app.CurrentVersionCode:
187 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188 apklist.append({'versionCode': int(build.versionCode),
189 'versionName': build.versionName,
190 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
195 if apk['versionCode'] == int(build.versionCode):
200 apklist.append({'versionCode': int(build.versionCode),
201 'versionName': build.versionName,
202 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
204 if app.CurrentVersionCode == '0':
206 # Sort with most recent first...
207 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
209 wikidata += "=Versions=\n"
210 if len(apklist) == 0:
211 wikidata += "We currently have no versions of this app available."
212 elif not gotcurrentver:
213 wikidata += "We don't have the current version of this app."
215 wikidata += "We have the current version of this app."
216 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218 if len(app.NoSourceSince) > 0:
219 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220 if len(app.CurrentVersion) > 0:
221 wikidata += "The current (recommended) version is " + app.CurrentVersion
222 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
225 wikidata += "==" + apk['versionName'] + "==\n"
227 if 'buildproblem' in apk:
228 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
231 wikidata += "This version is built and signed by "
233 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
235 wikidata += "the original developer.\n\n"
236 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
238 wikidata += '\n[[Category:' + wikicat + ']]\n'
239 if len(app.NoSourceSince) > 0:
240 wikidata += '\n[[Category:Apps missing source code]]\n'
241 if validapks == 0 and not app.Disabled:
242 wikidata += '\n[[Category:Apps with no packages]]\n'
243 if cantupdate and not app.Disabled:
244 wikidata += "\n[[Category:Apps we cannot update]]\n"
245 if buildfails and not app.Disabled:
246 wikidata += "\n[[Category:Apps with failing builds]]\n"
247 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248 wikidata += '\n[[Category:Apps to Update]]\n'
250 wikidata += '\n[[Category:Apps that are disabled]]\n'
251 if app.UpdateCheckMode == 'None' and not app.Disabled:
252 wikidata += '\n[[Category:Apps with no update check]]\n'
253 for appcat in app.Categories:
254 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
256 # We can't have underscores in the page name, even if they're in
257 # the package ID, because MediaWiki messes with them...
258 pagename = appid.replace('_', ' ')
260 # Drop a trailing newline, because mediawiki is going to drop it anyway
261 # and it we don't we'll think the page has changed when it hasn't...
262 if wikidata.endswith('\n'):
263 wikidata = wikidata[:-1]
265 generated_pages[pagename] = wikidata
267 # Make a redirect from the name to the ID too, unless there's
268 # already an existing page with the name and it isn't a redirect.
270 apppagename = app.Name.replace('_', ' ')
271 apppagename = apppagename.replace('{', '')
272 apppagename = apppagename.replace('}', ' ')
273 apppagename = apppagename.replace(':', ' ')
274 apppagename = apppagename.replace('[', ' ')
275 apppagename = apppagename.replace(']', ' ')
276 # Drop double spaces caused mostly by replacing ':' above
277 apppagename = apppagename.replace(' ', ' ')
278 for expagename in site.allpages(prefix=apppagename,
279 filterredir='nonredirects',
281 if expagename == apppagename:
283 # Another reason not to make the redirect page is if the app name
284 # is the same as it's ID, because that will overwrite the real page
285 # with an redirect to itself! (Although it seems like an odd
286 # scenario this happens a lot, e.g. where there is metadata but no
287 # builds or binaries to extract a name from.
288 if apppagename == pagename:
291 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
293 for tcat, genp in [(wikicat, generated_pages),
294 (wikiredircat, generated_redirects)]:
295 catpages = site.Pages['Category:' + tcat]
297 for page in catpages:
298 existingpages.append(page.name)
299 if page.name in genp:
300 pagetxt = page.edit()
301 if pagetxt != genp[page.name]:
302 logging.debug("Updating modified page " + page.name)
303 page.save(genp[page.name], summary='Auto-updated')
305 logging.debug("Page " + page.name + " is unchanged")
307 logging.warn("Deleting page " + page.name)
308 page.delete('No longer published')
309 for pagename, text in genp.items():
310 logging.debug("Checking " + pagename)
311 if pagename not in existingpages:
312 logging.debug("Creating page " + pagename)
314 newpage = site.Pages[pagename]
315 newpage.save(text, summary='Auto-created')
316 except Exception as e:
317 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
319 # Purge server cache to ensure counts are up to date
320 site.pages['Repository Maintenance'].purge()
323 def delete_disabled_builds(apps, apkcache, repodirs):
324 """Delete disabled build outputs.
326 :param apps: list of all applications, as per metadata.read_metadata
327 :param apkcache: current apk cache information
328 :param repodirs: the repo directories to process
330 for appid, app in apps.items():
331 for build in app['builds']:
332 if not build.disable:
334 apkfilename = common.get_release_filename(app, build)
335 iconfilename = "%s.%s.png" % (
338 for repodir in repodirs:
340 os.path.join(repodir, apkfilename),
341 os.path.join(repodir, apkfilename + '.asc'),
342 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
344 for density in all_screen_densities:
345 repo_dir = get_icon_dir(repodir, density)
346 files.append(os.path.join(repo_dir, iconfilename))
349 if os.path.exists(f):
350 logging.info("Deleting disabled build output " + f)
352 if apkfilename in apkcache:
353 del apkcache[apkfilename]
356 def resize_icon(iconpath, density):
358 if not os.path.isfile(iconpath):
363 fp = open(iconpath, 'rb')
365 size = dpi_to_px(density)
367 if any(length > size for length in im.size):
369 im.thumbnail((size, size), Image.ANTIALIAS)
370 logging.debug("%s was too large at %s - new size is %s" % (
371 iconpath, oldsize, im.size))
372 im.save(iconpath, "PNG")
374 except Exception as e:
375 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
382 def resize_all_icons(repodirs):
383 """Resize all icons that exceed the max size
385 :param repodirs: the repo directories to process
387 for repodir in repodirs:
388 for density in screen_densities:
389 icon_dir = get_icon_dir(repodir, density)
390 icon_glob = os.path.join(icon_dir, '*.png')
391 for iconpath in glob.glob(icon_glob):
392 resize_icon(iconpath, density)
396 """ Get the signing certificate of an apk. To get the same md5 has that
397 Android gets, we encode the .RSA certificate in a specific format and pass
398 it hex-encoded to the md5 digest algorithm.
400 :param apkpath: path to the apk
401 :returns: A string containing the md5 of the signature of the apk or None
402 if an error occurred.
405 with zipfile.ZipFile(apkpath, 'r') as apk:
406 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
409 logging.error("Found no signing certificates on %s" % apkpath)
412 logging.error("Found multiple signing certificates on %s" % apkpath)
415 cert = apk.read(certs[0])
417 cert_encoded = common.get_certificate(cert)
419 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
422 def get_cache_file():
423 return os.path.join('tmp', 'apkcache')
428 Gather information about all the apk files in the repo directory,
429 using cached data if possible.
432 apkcachefile = get_cache_file()
433 if not options.clean and os.path.exists(apkcachefile):
434 with open(apkcachefile, 'rb') as cf:
435 apkcache = pickle.load(cf, encoding='utf-8')
436 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
444 def write_cache(apkcache):
445 apkcachefile = get_cache_file()
446 cache_path = os.path.dirname(apkcachefile)
447 if not os.path.exists(cache_path):
448 os.makedirs(cache_path)
449 apkcache["METADATA_VERSION"] = METADATA_VERSION
450 with open(apkcachefile, 'wb') as cf:
451 pickle.dump(apkcache, cf)
454 def get_icon_bytes(apkzip, iconsrc):
455 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
457 return apkzip.read(iconsrc)
459 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
462 def sha256sum(filename):
463 '''Calculate the sha256 of the given file'''
464 sha = hashlib.sha256()
465 with open(filename, 'rb') as f:
471 return sha.hexdigest()
474 def has_old_openssl(filename):
475 '''checks for known vulnerable openssl versions in the APK'''
477 # statically load this pattern
478 if not hasattr(has_old_openssl, "pattern"):
479 has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
481 with zipfile.ZipFile(filename) as zf:
482 for name in zf.namelist():
483 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
486 chunk = lib.read(4096)
489 m = has_old_openssl.pattern.search(chunk)
491 version = m.group(1).decode('ascii')
492 if version.startswith('1.0.1') and version[5] >= 'r' \
493 or version.startswith('1.0.2') and version[5] >= 'f':
494 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
496 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
502 def insert_obbs(repodir, apps, apks):
503 """Scans the .obb files in a given repo directory and adds them to the
504 relevant APK instances. OBB files have versionCodes like APK
505 files, and they are loosely associated. If there is an OBB file
506 present, then any APK with the same or higher versionCode will use
507 that OBB file. There are two OBB types: main and patch, each APK
508 can only have only have one of each.
510 https://developer.android.com/google/play/expansion-files.html
512 :param repodir: repo directory to scan
513 :param apps: list of current, valid apps
514 :param apks: current information on all APKs
518 def obbWarnDelete(f, msg):
519 logging.warning(msg + f)
520 if options.delete_unknown:
521 logging.error("Deleting unknown file: " + f)
525 java_Integer_MIN_VALUE = -pow(2, 31)
526 currentPackageNames = apps.keys()
527 for f in glob.glob(os.path.join(repodir, '*.obb')):
528 obbfile = os.path.basename(f)
529 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
530 chunks = obbfile.split('.')
531 if chunks[0] != 'main' and chunks[0] != 'patch':
532 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
534 if not re.match(r'^-?[0-9]+$', chunks[1]):
535 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
537 versionCode = int(chunks[1])
538 packagename = ".".join(chunks[2:-1])
540 highestVersionCode = java_Integer_MIN_VALUE
541 if packagename not in currentPackageNames:
542 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
545 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
546 highestVersionCode = apk['versionCode']
547 if versionCode > highestVersionCode:
548 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
549 + ') than any APK: ')
551 obbsha256 = sha256sum(f)
552 obbs.append((packagename, versionCode, obbfile, obbsha256))
555 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
556 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
557 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
558 apk['obbMainFile'] = obbfile
559 apk['obbMainFileSha256'] = obbsha256
560 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
561 apk['obbPatchFile'] = obbfile
562 apk['obbPatchFileSha256'] = obbsha256
563 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
567 def _get_localized_dict(app, locale):
568 '''get the dict to add localized store metadata to'''
569 if 'localized' not in app:
570 app['localized'] = collections.OrderedDict()
571 if locale not in app['localized']:
572 app['localized'][locale] = collections.OrderedDict()
573 return app['localized'][locale]
576 def _set_localized_text_entry(app, locale, key, f):
577 limit = config['char_limits'][key]
578 localized = _get_localized_dict(app, locale)
580 text = fp.read()[:limit]
582 localized[key] = text
585 def _set_author_entry(app, key, f):
586 limit = config['char_limits']['author']
588 text = fp.read()[:limit]
593 def copy_triple_t_store_metadata(apps):
594 """Include store metadata from the app's source repo
596 The Triple-T Gradle Play Publisher is a plugin that has a standard
597 file layout for all of the metadata and graphics that the Google
598 Play Store accepts. Since F-Droid has the git repo, it can just
599 pluck those files directly. This method reads any text files into
600 the app dict, then copies any graphics into the fdroid repo
603 This needs to be run before insert_localized_app_metadata() so that
604 the graphics files that are copied into the fdroid repo get
607 https://github.com/Triple-T/gradle-play-publisher#upload-images
608 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
612 if not os.path.isdir('build'):
613 return # nothing to do
615 for packageName, app in apps.items():
616 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
617 logging.debug('Triple-T Gradle Play Publisher: ' + d)
618 for root, dirs, files in os.walk(d):
619 segments = root.split('/')
620 locale = segments[-2]
622 if f == 'fulldescription':
623 _set_localized_text_entry(app, locale, 'description',
624 os.path.join(root, f))
626 elif f == 'shortdescription':
627 _set_localized_text_entry(app, locale, 'summary',
628 os.path.join(root, f))
631 _set_localized_text_entry(app, locale, 'name',
632 os.path.join(root, f))
635 _set_localized_text_entry(app, locale, 'video',
636 os.path.join(root, f))
638 elif f == 'whatsnew':
639 _set_localized_text_entry(app, segments[-1], 'whatsNew',
640 os.path.join(root, f))
642 elif f == 'contactEmail':
643 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
645 elif f == 'contactPhone':
646 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
648 elif f == 'contactWebsite':
649 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
652 base, extension = common.get_extension(f)
653 dirname = os.path.basename(root)
654 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
655 if segments[-2] == 'listing':
656 locale = segments[-3]
658 locale = segments[-2]
659 destdir = os.path.join('repo', packageName, locale)
660 os.makedirs(destdir, mode=0o755, exist_ok=True)
661 sourcefile = os.path.join(root, f)
662 destfile = os.path.join(destdir, dirname + '.' + extension)
663 logging.debug('copying ' + sourcefile + ' ' + destfile)
664 shutil.copy(sourcefile, destfile)
667 def insert_localized_app_metadata(apps):
668 """scans standard locations for graphics and localized text
670 Scans for localized description files, store graphics, and
671 screenshot PNG files in statically defined screenshots directory
672 and adds them to the app metadata. The screenshots and graphic
673 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
674 and must be in the following layout:
675 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
677 repo/packageName/locale/featureGraphic.png
678 repo/packageName/locale/phoneScreenshots/1.png
679 repo/packageName/locale/phoneScreenshots/2.png
681 The changelog files must be text files named with the versionCode
682 ending with ".txt" and must be in the following layout:
683 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
685 repo/packageName/locale/changelogs/12345.txt
687 This will scan the each app's source repo then the metadata/ dir
688 for these standard locations of changelog files. If it finds
689 them, they will be added to the dict of all packages, with the
690 versions in the metadata/ folder taking precendence over the what
691 is in the app's source repo.
693 Where "packageName" is the app's packageName and "locale" is the locale
694 of the graphics, e.g. what language they are in, using the IETF RFC5646
695 format (en-US, fr-CA, es-MX, etc).
697 This will also scan the app's git for a fastlane folder, and the
698 metadata/ folder and the apps' source repos for standard locations
699 of graphic and screenshot files. If it finds them, it will copy
700 them into the repo. The fastlane files follow this pattern:
701 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
705 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
706 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
707 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
709 for d in sorted(sourcedirs):
710 if not os.path.isdir(d):
712 for root, dirs, files in os.walk(d):
713 segments = root.split('/')
714 packageName = segments[1]
715 if packageName not in apps:
716 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
718 locale = segments[-1]
720 if f in ('description.txt', 'full_description.txt'):
721 _set_localized_text_entry(apps[packageName], locale, 'description',
722 os.path.join(root, f))
724 elif f in ('summary.txt', 'short_description.txt'):
725 _set_localized_text_entry(apps[packageName], locale, 'summary',
726 os.path.join(root, f))
728 elif f in ('name.txt', 'title.txt'):
729 _set_localized_text_entry(apps[packageName], locale, 'name',
730 os.path.join(root, f))
732 elif f == 'video.txt':
733 _set_localized_text_entry(apps[packageName], locale, 'video',
734 os.path.join(root, f))
736 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
737 locale = segments[-2]
738 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
739 os.path.join(root, f))
742 base, extension = common.get_extension(f)
743 if locale == 'images':
744 locale = segments[-2]
745 destdir = os.path.join('repo', packageName, locale)
746 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
747 os.makedirs(destdir, mode=0o755, exist_ok=True)
748 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
749 shutil.copy(os.path.join(root, f), destdir)
751 if d in SCREENSHOT_DIRS:
752 for f in glob.glob(os.path.join(root, d, '*.*')):
753 _, extension = common.get_extension(f)
754 if extension in ALLOWED_EXTENSIONS:
755 screenshotdestdir = os.path.join(destdir, d)
756 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
757 logging.debug('copying ' + f + ' ' + screenshotdestdir)
758 shutil.copy(f, screenshotdestdir)
760 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
762 if not os.path.isdir(d):
764 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
765 if not os.path.isfile(f):
767 segments = f.split('/')
768 packageName = segments[1]
770 screenshotdir = segments[3]
771 filename = os.path.basename(f)
772 base, extension = common.get_extension(filename)
774 if packageName not in apps:
775 logging.warning('Found "%s" graphic without metadata for app "%s"!'
776 % (filename, packageName))
778 graphics = _get_localized_dict(apps[packageName], locale)
780 if extension not in ALLOWED_EXTENSIONS:
781 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
782 elif base in GRAPHIC_NAMES:
783 # there can only be zero or one of these per locale
784 graphics[base] = filename
785 elif screenshotdir in SCREENSHOT_DIRS:
786 # there can any number of these per locale
787 logging.debug('adding to ' + screenshotdir + ': ' + f)
788 if screenshotdir not in graphics:
789 graphics[screenshotdir] = []
790 graphics[screenshotdir].append(filename)
792 logging.warning('Unsupported graphics file found: ' + f)
795 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
796 """Scan a repo for all files with an extension except APK/OBB
798 :param apkcache: current cached info about all repo files
799 :param repodir: repo directory to scan
800 :param knownapks: list of all known files, as per metadata.read_metadata
801 :param use_date_from_file: use date from file (instead of current date)
802 for newly added files
807 repodir = repodir.encode('utf-8')
808 for name in os.listdir(repodir):
809 file_extension = common.get_file_extension(name)
810 if file_extension == 'apk' or file_extension == 'obb':
812 filename = os.path.join(repodir, name)
813 name_utf8 = name.decode('utf-8')
814 if filename.endswith(b'_src.tar.gz'):
815 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
817 if not common.is_repo_file(filename):
819 stat = os.stat(filename)
820 if stat.st_size == 0:
821 raise FDroidException(filename + ' is zero size!')
823 shasum = sha256sum(filename)
826 repo_file = apkcache[name]
827 # added time is cached as tuple but used here as datetime instance
828 if 'added' in repo_file:
829 a = repo_file['added']
830 if isinstance(a, datetime):
831 repo_file['added'] = a
833 repo_file['added'] = datetime(*a[:6])
834 if repo_file.get('hash') == shasum:
835 logging.debug("Reading " + name_utf8 + " from cache")
838 logging.debug("Ignoring stale cache data for " + name)
841 logging.debug("Processing " + name_utf8)
842 repo_file = collections.OrderedDict()
843 repo_file['name'] = os.path.splitext(name_utf8)[0]
844 # TODO rename apkname globally to something more generic
845 repo_file['apkName'] = name_utf8
846 repo_file['hash'] = shasum
847 repo_file['hashType'] = 'sha256'
848 repo_file['versionCode'] = 0
849 repo_file['versionName'] = shasum
850 # the static ID is the SHA256 unless it is set in the metadata
851 repo_file['packageName'] = shasum
853 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
855 repo_file['packageName'] = m.group(1)
856 repo_file['versionCode'] = int(m.group(2))
857 srcfilename = name + b'_src.tar.gz'
858 if os.path.exists(os.path.join(repodir, srcfilename)):
859 repo_file['srcname'] = srcfilename.decode('utf-8')
860 repo_file['size'] = stat.st_size
862 apkcache[name] = repo_file
865 if use_date_from_file:
866 timestamp = stat.st_ctime
867 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
869 default_date_param = None
871 # Record in knownapks, getting the added date at the same time..
872 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
873 default_date=default_date_param)
875 repo_file['added'] = added
877 repo_files.append(repo_file)
879 return repo_files, cachechanged
882 def scan_apk_aapt(apk, apkfile):
883 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
884 if p.returncode != 0:
885 if options.delete_unknown:
886 if os.path.exists(apkfile):
887 logging.error("Failed to get apk information, deleting " + apkfile)
890 logging.error("Could not find {0} to remove it".format(apkfile))
892 logging.error("Failed to get apk information, skipping " + apkfile)
893 raise BuildException("Invalid APK")
894 for line in p.output.splitlines():
895 if line.startswith("package:"):
897 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
898 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
899 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
900 except Exception as e:
901 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
902 elif line.startswith("application:"):
903 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
904 # Keep path to non-dpi icon in case we need it
905 match = re.match(APK_ICON_PAT_NODPI, line)
907 apk['icons_src']['-1'] = match.group(1)
908 elif line.startswith("launchable-activity:"):
909 # Only use launchable-activity as fallback to application
911 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
912 if '-1' not in apk['icons_src']:
913 match = re.match(APK_ICON_PAT_NODPI, line)
915 apk['icons_src']['-1'] = match.group(1)
916 elif line.startswith("application-icon-"):
917 match = re.match(APK_ICON_PAT, line)
919 density = match.group(1)
920 path = match.group(2)
921 apk['icons_src'][density] = path
922 elif line.startswith("sdkVersion:"):
923 m = re.match(APK_SDK_VERSION_PAT, line)
925 logging.error(line.replace('sdkVersion:', '')
926 + ' is not a valid minSdkVersion!')
928 apk['minSdkVersion'] = m.group(1)
929 # if target not set, default to min
930 if 'targetSdkVersion' not in apk:
931 apk['targetSdkVersion'] = m.group(1)
932 elif line.startswith("targetSdkVersion:"):
933 m = re.match(APK_SDK_VERSION_PAT, line)
935 logging.error(line.replace('targetSdkVersion:', '')
936 + ' is not a valid targetSdkVersion!')
938 apk['targetSdkVersion'] = m.group(1)
939 elif line.startswith("maxSdkVersion:"):
940 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
941 elif line.startswith("native-code:"):
942 apk['nativecode'] = []
943 for arch in line[13:].split(' '):
944 apk['nativecode'].append(arch[1:-1])
945 elif line.startswith('uses-permission:'):
946 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
947 if perm_match['maxSdkVersion']:
948 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
949 permission = UsesPermission(
951 perm_match['maxSdkVersion']
954 apk['uses-permission'].append(permission)
955 elif line.startswith('uses-permission-sdk-23:'):
956 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
957 if perm_match['maxSdkVersion']:
958 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
959 permission_sdk_23 = UsesPermissionSdk23(
961 perm_match['maxSdkVersion']
964 apk['uses-permission-sdk-23'].append(permission_sdk_23)
966 elif line.startswith('uses-feature:'):
967 feature = re.match(APK_FEATURE_PAT, line).group(1)
968 # Filter out this, it's only added with the latest SDK tools and
969 # causes problems for lots of apps.
970 if feature != "android.hardware.screen.portrait" \
971 and feature != "android.hardware.screen.landscape":
972 if feature.startswith("android.feature."):
973 feature = feature[16:]
974 apk['features'].add(feature)
977 def scan_apk_androguard(apk, apkfile):
979 from androguard.core.bytecodes.apk import APK
980 apkobject = APK(apkfile)
981 if apkobject.is_valid_APK():
982 arsc = apkobject.get_android_resources()
984 if options.delete_unknown:
985 if os.path.exists(apkfile):
986 logging.error("Failed to get apk information, deleting " + apkfile)
989 logging.error("Could not find {0} to remove it".format(apkfile))
991 logging.error("Failed to get apk information, skipping " + apkfile)
992 raise BuildException("Invaild APK")
994 raise FDroidException("androguard library is not installed and aapt not present")
995 except FileNotFoundError:
996 logging.error("Could not open apk file for analysis")
997 raise BuildException("Invalid APK")
999 apk['packageName'] = apkobject.get_package()
1000 apk['versionCode'] = int(apkobject.get_androidversion_code())
1001 apk['versionName'] = apkobject.get_androidversion_name()
1002 if apk['versionName'][0] == "@":
1003 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1004 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1005 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1006 apk['name'] = apkobject.get_app_name()
1008 if apkobject.get_max_sdk_version() is not None:
1009 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1010 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1011 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1013 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1014 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1016 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1018 for file in apkobject.get_files():
1019 d_re = density_re.match(file)
1021 folder = d_re.group(1).split('-')
1023 resolution = folder[1]
1026 density = screen_resolutions[resolution]
1027 apk['icons_src'][density] = d_re.group(0)
1029 if apk['icons_src'].get('-1') is None:
1030 apk['icons_src']['-1'] = apk['icons_src']['160']
1032 arch_re = re.compile("^lib/(.*)/.*$")
1033 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1035 apk['nativecode'] = []
1036 apk['nativecode'].extend(sorted(list(arch)))
1038 xml = apkobject.get_android_manifest_xml()
1040 for item in xml.getElementsByTagName('uses-permission'):
1041 name = str(item.getAttribute("android:name"))
1042 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1043 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1044 permission = UsesPermission(
1048 apk['uses-permission'].append(permission)
1050 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1051 name = str(item.getAttribute("android:name"))
1052 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1053 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1054 permission_sdk_23 = UsesPermissionSdk23(
1058 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1060 for item in xml.getElementsByTagName('uses-feature'):
1061 feature = str(item.getAttribute("android:name"))
1062 if feature != "android.hardware.screen.portrait" \
1063 and feature != "android.hardware.screen.landscape":
1064 if feature.startswith("android.feature."):
1065 feature = feature[16:]
1066 apk['features'].append(feature)
1069 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1070 """Scan the apk with the given filename in the given repo directory.
1072 This also extracts the icons.
1074 :param apkcache: current apk cache information
1075 :param apkfilename: the filename of the apk to scan
1076 :param repodir: repo directory to scan
1077 :param knownapks: known apks info
1078 :param use_date_from_apk: use date from APK (instead of current date)
1079 for newly added APKs
1080 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1081 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1084 if ' ' in apkfilename:
1085 if options.rename_apks:
1086 newfilename = apkfilename.replace(' ', '_')
1087 os.rename(os.path.join(repodir, apkfilename),
1088 os.path.join(repodir, newfilename))
1089 apkfilename = newfilename
1091 logging.critical("Spaces in filenames are not allowed.")
1092 return True, None, False
1094 apkfile = os.path.join(repodir, apkfilename)
1095 shasum = sha256sum(apkfile)
1097 cachechanged = False
1099 if apkfilename in apkcache:
1100 apk = apkcache[apkfilename]
1101 if apk.get('hash') == shasum:
1102 logging.debug("Reading " + apkfilename + " from cache")
1105 logging.debug("Ignoring stale cache data for " + apkfilename)
1108 logging.debug("Processing " + apkfilename)
1110 apk['hash'] = shasum
1111 apk['hashType'] = 'sha256'
1112 apk['uses-permission'] = []
1113 apk['uses-permission-sdk-23'] = []
1114 apk['features'] = []
1115 apk['icons_src'] = {}
1117 apk['antiFeatures'] = set()
1120 if SdkToolsPopen(['aapt', 'version'], output=False):
1121 scan_apk_aapt(apk, apkfile)
1123 scan_apk_androguard(apk, apkfile)
1124 except BuildException:
1125 return True, None, False
1127 if 'minSdkVersion' not in apk:
1128 logging.warn("No SDK version information found in {0}".format(apkfile))
1129 apk['minSdkVersion'] = 1
1131 # Check for debuggable apks...
1132 if common.isApkAndDebuggable(apkfile):
1133 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1135 # Get the signature (or md5 of, to be precise)...
1136 logging.debug('Getting signature of {0}'.format(apkfile))
1137 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1139 logging.critical("Failed to get apk signature")
1140 return True, None, False
1142 if options.rename_apks:
1143 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1144 std_short_name = os.path.join(repodir, n)
1145 if apkfile != std_short_name:
1146 if os.path.exists(std_short_name):
1147 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1148 if apkfile != std_long_name:
1149 if os.path.exists(std_long_name):
1150 dupdir = os.path.join('duplicates', repodir)
1151 if not os.path.isdir(dupdir):
1152 os.makedirs(dupdir, exist_ok=True)
1153 dupfile = os.path.join('duplicates', std_long_name)
1154 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1155 os.rename(apkfile, dupfile)
1156 return True, None, False
1158 os.rename(apkfile, std_long_name)
1159 apkfile = std_long_name
1161 os.rename(apkfile, std_short_name)
1162 apkfile = std_short_name
1163 apkfilename = apkfile[len(repodir) + 1:]
1165 apk['apkName'] = apkfilename
1166 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1167 if os.path.exists(os.path.join(repodir, srcfilename)):
1168 apk['srcname'] = srcfilename
1169 apk['size'] = os.path.getsize(apkfile)
1171 # verify the jar signature is correct
1172 if not common.verify_apk_signature(apkfile):
1173 return True, None, False
1175 if has_old_openssl(apkfile):
1176 apk['antiFeatures'].add('KnownVuln')
1178 apkzip = zipfile.ZipFile(apkfile, 'r')
1180 # if an APK has files newer than the system time, suggest updating
1181 # the system clock. This is useful for offline systems, used for
1182 # signing, which do not have another source of clock sync info. It
1183 # has to be more than 24 hours newer because ZIP/APK files do not
1184 # store timezone info
1185 manifest = apkzip.getinfo('AndroidManifest.xml')
1186 if manifest.date_time[1] == 0: # month can't be zero
1187 logging.debug('AndroidManifest.xml has no date')
1189 dt_obj = datetime(*manifest.date_time)
1190 checkdt = dt_obj - timedelta(1)
1191 if datetime.today() < checkdt:
1192 logging.warn('System clock is older than manifest in: '
1194 + '\nSet clock to that time using:\n'
1195 + 'sudo date -s "' + str(dt_obj) + '"')
1197 iconfilename = "%s.%s.png" % (
1201 # Extract the icon file...
1202 empty_densities = []
1203 for density in screen_densities:
1204 if density not in apk['icons_src']:
1205 empty_densities.append(density)
1207 iconsrc = apk['icons_src'][density]
1208 icon_dir = get_icon_dir(repodir, density)
1209 icondest = os.path.join(icon_dir, iconfilename)
1212 with open(icondest, 'wb') as f:
1213 f.write(get_icon_bytes(apkzip, iconsrc))
1214 apk['icons'][density] = iconfilename
1215 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1216 logging.warning("Error retrieving icon file: %s" % (icondest))
1217 del apk['icons_src'][density]
1218 empty_densities.append(density)
1220 if '-1' in apk['icons_src']:
1221 iconsrc = apk['icons_src']['-1']
1222 iconpath = os.path.join(
1223 get_icon_dir(repodir, '0'), iconfilename)
1224 with open(iconpath, 'wb') as f:
1225 f.write(get_icon_bytes(apkzip, iconsrc))
1227 im = Image.open(iconpath)
1228 dpi = px_to_dpi(im.size[0])
1229 for density in screen_densities:
1230 if density in apk['icons']:
1232 if density == screen_densities[-1] or dpi >= int(density):
1233 apk['icons'][density] = iconfilename
1234 shutil.move(iconpath,
1235 os.path.join(get_icon_dir(repodir, density), iconfilename))
1236 empty_densities.remove(density)
1238 except Exception as e:
1239 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1242 apk['icon'] = iconfilename
1246 # First try resizing down to not lose quality
1248 for density in screen_densities:
1249 if density not in empty_densities:
1250 last_density = density
1252 if last_density is None:
1254 logging.debug("Density %s not available, resizing down from %s"
1255 % (density, last_density))
1257 last_iconpath = os.path.join(
1258 get_icon_dir(repodir, last_density), iconfilename)
1259 iconpath = os.path.join(
1260 get_icon_dir(repodir, density), iconfilename)
1263 fp = open(last_iconpath, 'rb')
1266 size = dpi_to_px(density)
1268 im.thumbnail((size, size), Image.ANTIALIAS)
1269 im.save(iconpath, "PNG")
1270 empty_densities.remove(density)
1271 except Exception as e:
1272 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1277 # Then just copy from the highest resolution available
1279 for density in reversed(screen_densities):
1280 if density not in empty_densities:
1281 last_density = density
1283 if last_density is None:
1285 logging.debug("Density %s not available, copying from lower density %s"
1286 % (density, last_density))
1289 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1290 os.path.join(get_icon_dir(repodir, density), iconfilename))
1292 empty_densities.remove(density)
1294 for density in screen_densities:
1295 icon_dir = get_icon_dir(repodir, density)
1296 icondest = os.path.join(icon_dir, iconfilename)
1297 resize_icon(icondest, density)
1299 # Copy from icons-mdpi to icons since mdpi is the baseline density
1300 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1301 if os.path.isfile(baseline):
1302 apk['icons']['0'] = iconfilename
1303 shutil.copyfile(baseline,
1304 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1306 if use_date_from_apk and manifest.date_time[1] != 0:
1307 default_date_param = datetime(*manifest.date_time)
1309 default_date_param = None
1311 # Record in known apks, getting the added date at the same time..
1312 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1313 default_date=default_date_param)
1315 apk['added'] = added
1317 apkcache[apkfilename] = apk
1320 return False, apk, cachechanged
1323 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1324 """Scan the apks in the given repo directory.
1326 This also extracts the icons.
1328 :param apkcache: current apk cache information
1329 :param repodir: repo directory to scan
1330 :param knownapks: known apks info
1331 :param use_date_from_apk: use date from APK (instead of current date)
1332 for newly added APKs
1333 :returns: (apks, cachechanged) where apks is a list of apk information,
1334 and cachechanged is True if the apkcache got changed.
1337 cachechanged = False
1339 for icon_dir in get_all_icon_dirs(repodir):
1340 if os.path.exists(icon_dir):
1342 shutil.rmtree(icon_dir)
1343 os.makedirs(icon_dir)
1345 os.makedirs(icon_dir)
1348 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1349 apkfilename = apkfile[len(repodir) + 1:]
1350 (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1355 return apks, cachechanged
1358 def apply_info_from_latest_apk(apps, apks):
1360 Some information from the apks needs to be applied up to the application level.
1361 When doing this, we use the info from the most recent version's apk.
1362 We deal with figuring out when the app was added and last updated at the same time.
1364 for appid, app in apps.items():
1365 bestver = UNSET_VERSION_CODE
1367 if apk['packageName'] == appid:
1368 if apk['versionCode'] > bestver:
1369 bestver = apk['versionCode']
1373 if not app.added or apk['added'] < app.added:
1374 app.added = apk['added']
1375 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1376 app.lastUpdated = apk['added']
1379 logging.debug("Don't know when " + appid + " was added")
1380 if not app.lastUpdated:
1381 logging.debug("Don't know when " + appid + " was last updated")
1383 if bestver == UNSET_VERSION_CODE:
1385 if app.Name is None:
1386 app.Name = app.AutoName or appid
1388 logging.debug("Application " + appid + " has no packages")
1390 if app.Name is None:
1391 app.Name = bestapk['name']
1392 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1393 if app.CurrentVersionCode is None:
1394 app.CurrentVersionCode = str(bestver)
1397 def make_categories_txt(repodir, categories):
1398 '''Write a category list in the repo to allow quick access'''
1400 for cat in sorted(categories):
1401 catdata += cat + '\n'
1402 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1406 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1408 for appid, app in apps.items():
1410 if app.ArchivePolicy:
1411 keepversions = int(app.ArchivePolicy[:-9])
1413 keepversions = defaultkeepversions
1415 def filter_apk_list_sorted(apk_list):
1417 for apk in apk_list:
1418 if apk['packageName'] == appid:
1421 # Sort the apk list by version code. First is highest/newest.
1422 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1424 def move_file(from_dir, to_dir, filename, ignore_missing):
1425 from_path = os.path.join(from_dir, filename)
1426 if ignore_missing and not os.path.exists(from_path):
1428 to_path = os.path.join(to_dir, filename)
1429 shutil.move(from_path, to_path)
1431 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1432 .format(appid, len(apks), keepversions, len(archapks)))
1434 if len(apks) > keepversions:
1435 apklist = filter_apk_list_sorted(apks)
1436 # Move back the ones we don't want.
1437 for apk in apklist[keepversions:]:
1438 logging.info("Moving " + apk['apkName'] + " to archive")
1439 move_file(repodir, archivedir, apk['apkName'], False)
1440 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1441 for density in all_screen_densities:
1442 repo_icon_dir = get_icon_dir(repodir, density)
1443 archive_icon_dir = get_icon_dir(archivedir, density)
1444 if density not in apk['icons']:
1446 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1447 if 'srcname' in apk:
1448 move_file(repodir, archivedir, apk['srcname'], False)
1449 archapks.append(apk)
1451 elif len(apks) < keepversions and len(archapks) > 0:
1452 required = keepversions - len(apks)
1453 archapklist = filter_apk_list_sorted(archapks)
1454 # Move forward the ones we want again.
1455 for apk in archapklist[:required]:
1456 logging.info("Moving " + apk['apkName'] + " from archive")
1457 move_file(archivedir, repodir, apk['apkName'], False)
1458 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1459 for density in all_screen_densities:
1460 repo_icon_dir = get_icon_dir(repodir, density)
1461 archive_icon_dir = get_icon_dir(archivedir, density)
1462 if density not in apk['icons']:
1464 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1465 if 'srcname' in apk:
1466 move_file(archivedir, repodir, apk['srcname'], False)
1467 archapks.remove(apk)
1471 def add_apks_to_per_app_repos(repodir, apks):
1472 apks_per_app = dict()
1474 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1475 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1476 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1477 apks_per_app[apk['packageName']] = apk
1479 if not os.path.exists(apk['per_app_icons']):
1480 logging.info('Adding new repo for only ' + apk['packageName'])
1481 os.makedirs(apk['per_app_icons'])
1483 apkpath = os.path.join(repodir, apk['apkName'])
1484 shutil.copy(apkpath, apk['per_app_repo'])
1485 apksigpath = apkpath + '.sig'
1486 if os.path.exists(apksigpath):
1487 shutil.copy(apksigpath, apk['per_app_repo'])
1488 apkascpath = apkpath + '.asc'
1489 if os.path.exists(apkascpath):
1490 shutil.copy(apkascpath, apk['per_app_repo'])
1499 global config, options
1501 # Parse command line...
1502 parser = ArgumentParser()
1503 common.setup_global_opts(parser)
1504 parser.add_argument("--create-key", action="store_true", default=False,
1505 help="Create a repo signing key in a keystore")
1506 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1507 help="Create skeleton metadata files that are missing")
1508 parser.add_argument("--delete-unknown", action="store_true", default=False,
1509 help="Delete APKs and/or OBBs without metadata from the repo")
1510 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1511 help="Report on build data status")
1512 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1513 help="Interactively ask about things that need updating.")
1514 parser.add_argument("-I", "--icons", action="store_true", default=False,
1515 help="Resize all the icons exceeding the max pixel size and exit")
1516 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1517 help="Specify editor to use in interactive mode. Default " +
1518 "is /etc/alternatives/editor")
1519 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1520 help="Update the wiki")
1521 parser.add_argument("--pretty", action="store_true", default=False,
1522 help="Produce human-readable index.xml")
1523 parser.add_argument("--clean", action="store_true", default=False,
1524 help="Clean update - don't uses caches, reprocess all apks")
1525 parser.add_argument("--nosign", action="store_true", default=False,
1526 help="When configured for signed indexes, create only unsigned indexes at this stage")
1527 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1528 help="Use date from apk instead of current time for newly added apks")
1529 parser.add_argument("--rename-apks", action="store_true", default=False,
1530 help="Rename APK files that do not match package.name_123.apk")
1531 metadata.add_metadata_arguments(parser)
1532 options = parser.parse_args()
1533 metadata.warnings_action = options.W
1535 config = common.read_config(options)
1537 if not ('jarsigner' in config and 'keytool' in config):
1538 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1541 if config['archive_older'] != 0:
1542 repodirs.append('archive')
1543 if not os.path.exists('archive'):
1547 resize_all_icons(repodirs)
1550 if options.rename_apks:
1551 options.clean = True
1553 # check that icons exist now, rather than fail at the end of `fdroid update`
1554 for k in ['repo_icon', 'archive_icon']:
1556 if not os.path.exists(config[k]):
1557 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1560 # if the user asks to create a keystore, do it now, reusing whatever it can
1561 if options.create_key:
1562 if os.path.exists(config['keystore']):
1563 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1564 logging.critical("\t'" + config['keystore'] + "'")
1567 if 'repo_keyalias' not in config:
1568 config['repo_keyalias'] = socket.getfqdn()
1569 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1570 if 'keydname' not in config:
1571 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1572 common.write_to_config(config, 'keydname', config['keydname'])
1573 if 'keystore' not in config:
1574 config['keystore'] = common.default_config['keystore']
1575 common.write_to_config(config, 'keystore', config['keystore'])
1577 password = common.genpassword()
1578 if 'keystorepass' not in config:
1579 config['keystorepass'] = password
1580 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1581 if 'keypass' not in config:
1582 config['keypass'] = password
1583 common.write_to_config(config, 'keypass', config['keypass'])
1584 common.genkeystore(config)
1587 apps = metadata.read_metadata()
1589 # Generate a list of categories...
1591 for app in apps.values():
1592 categories.update(app.Categories)
1594 # Read known apks data (will be updated and written back when we've finished)
1595 knownapks = common.KnownApks()
1598 apkcache = get_cache()
1600 # Delete builds for disabled apps
1601 delete_disabled_builds(apps, apkcache, repodirs)
1603 # Scan all apks in the main repo
1604 apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1606 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1607 options.use_date_from_apk)
1608 cachechanged = cachechanged or fcachechanged
1610 # Generate warnings for apk's with no metadata (or create skeleton
1611 # metadata files, if requested on the command line)
1614 if apk['packageName'] not in apps:
1615 if options.create_metadata:
1616 if 'name' not in apk:
1617 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1619 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1620 f.write("License:Unknown\n")
1621 f.write("Web Site:\n")
1622 f.write("Source Code:\n")
1623 f.write("Issue Tracker:\n")
1624 f.write("Changelog:\n")
1625 f.write("Summary:" + apk['name'] + "\n")
1626 f.write("Description:\n")
1627 f.write(apk['name'] + "\n")
1629 f.write("Name:" + apk['name'] + "\n")
1631 logging.info("Generated skeleton metadata for " + apk['packageName'])
1634 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1635 if options.delete_unknown:
1636 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1637 rmf = os.path.join(repodirs[0], apk['apkName'])
1638 if not os.path.exists(rmf):
1639 logging.error("Could not find {0} to remove it".format(rmf))
1643 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1645 # update the metadata with the newly created ones included
1647 apps = metadata.read_metadata()
1649 copy_triple_t_store_metadata(apps)
1650 insert_obbs(repodirs[0], apps, apks)
1651 insert_localized_app_metadata(apps)
1653 # Scan the archive repo for apks as well
1654 if len(repodirs) > 1:
1655 archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1661 # Apply information from latest apks to the application and update dates
1662 apply_info_from_latest_apk(apps, apks + archapks)
1664 # Sort the app list by name, then the web site doesn't have to by default.
1665 # (we had to wait until we'd scanned the apks to do this, because mostly the
1666 # name comes from there!)
1667 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1669 # APKs are placed into multiple repos based on the app package, providing
1670 # per-app subscription feeds for nightly builds and things like it
1671 if config['per_app_repos']:
1672 add_apks_to_per_app_repos(repodirs[0], apks)
1673 for appid, app in apps.items():
1674 repodir = os.path.join(appid, 'fdroid', 'repo')
1676 appdict[appid] = app
1677 if os.path.isdir(repodir):
1678 index.make(appdict, [appid], apks, repodir, False)
1680 logging.info('Skipping index generation for ' + appid)
1683 if len(repodirs) > 1:
1684 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1686 # Make the index for the main repo...
1687 index.make(apps, sortedids, apks, repodirs[0], False)
1688 make_categories_txt(repodirs[0], categories)
1690 # If there's an archive repo, make the index for it. We already scanned it
1692 if len(repodirs) > 1:
1693 index.make(apps, sortedids, archapks, repodirs[1], True)
1695 git_remote = config.get('binary_transparency_remote')
1696 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1697 btlog.make_binary_transparency_log(repodirs)
1699 if config['update_stats']:
1700 # Update known apks info...
1701 knownapks.writeifchanged()
1703 # Generate latest apps data for widget
1704 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1706 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1708 appid = line.rstrip()
1709 data += appid + "\t"
1711 data += app.Name + "\t"
1712 if app.icon is not None:
1713 data += app.icon + "\t"
1714 data += app.License + "\n"
1715 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1719 write_cache(apkcache)
1721 # Update the wiki...
1723 update_wiki(apps, sortedids, apks + archapks)
1725 logging.info("Finished.")
1728 if __name__ == "__main__":