3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
76 all_screen_densities = ['0'] + screen_densities
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 def dpi_to_px(density):
88 return (int(density) * 48) / 160
92 return (int(px) * 160) / 48
95 def get_icon_dir(repodir, density):
97 return os.path.join(repodir, "icons")
98 return os.path.join(repodir, "icons-%s" % density)
101 def get_icon_dirs(repodir):
102 for density in screen_densities:
103 yield get_icon_dir(repodir, density)
106 def get_all_icon_dirs(repodir):
107 for density in all_screen_densities:
108 yield get_icon_dir(repodir, density)
111 def update_wiki(apps, sortedids, apks):
114 :param apps: fully populated list of all applications
115 :param apks: all apks, except...
117 logging.info("Updating wiki")
119 wikiredircat = 'App Redirects'
121 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122 path=config['wiki_path'])
123 site.login(config['wiki_user'], config['wiki_password'])
125 generated_redirects = {}
127 for appid in sortedids:
128 app = metadata.App(apps[appid])
132 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
134 for af in app.AntiFeatures:
135 wikidata += '{{AntiFeature|' + af + '}}\n'
140 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
143 app.added.strftime('%Y-%m-%d') if app.added else '',
144 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
159 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
161 wikidata += app.Summary
162 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
164 wikidata += "=Description=\n"
165 wikidata += metadata.description_wiki(app.Description) + "\n"
167 wikidata += "=Maintainer Notes=\n"
168 if app.MaintainerNotes:
169 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
172 # Get a list of all packages for this application...
174 gotcurrentver = False
178 if apk['packageName'] == appid:
179 if str(apk['versionCode']) == app.CurrentVersionCode:
182 # Include ones we can't build, as a special case...
183 for build in app.builds:
185 if build.versionCode == app.CurrentVersionCode:
187 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188 apklist.append({'versionCode': int(build.versionCode),
189 'versionName': build.versionName,
190 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
195 if apk['versionCode'] == int(build.versionCode):
200 apklist.append({'versionCode': int(build.versionCode),
201 'versionName': build.versionName,
202 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
204 if app.CurrentVersionCode == '0':
206 # Sort with most recent first...
207 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
209 wikidata += "=Versions=\n"
210 if len(apklist) == 0:
211 wikidata += "We currently have no versions of this app available."
212 elif not gotcurrentver:
213 wikidata += "We don't have the current version of this app."
215 wikidata += "We have the current version of this app."
216 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218 if len(app.NoSourceSince) > 0:
219 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220 if len(app.CurrentVersion) > 0:
221 wikidata += "The current (recommended) version is " + app.CurrentVersion
222 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
225 wikidata += "==" + apk['versionName'] + "==\n"
227 if 'buildproblem' in apk:
228 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
231 wikidata += "This version is built and signed by "
233 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
235 wikidata += "the original developer.\n\n"
236 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
238 wikidata += '\n[[Category:' + wikicat + ']]\n'
239 if len(app.NoSourceSince) > 0:
240 wikidata += '\n[[Category:Apps missing source code]]\n'
241 if validapks == 0 and not app.Disabled:
242 wikidata += '\n[[Category:Apps with no packages]]\n'
243 if cantupdate and not app.Disabled:
244 wikidata += "\n[[Category:Apps we cannot update]]\n"
245 if buildfails and not app.Disabled:
246 wikidata += "\n[[Category:Apps with failing builds]]\n"
247 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248 wikidata += '\n[[Category:Apps to Update]]\n'
250 wikidata += '\n[[Category:Apps that are disabled]]\n'
251 if app.UpdateCheckMode == 'None' and not app.Disabled:
252 wikidata += '\n[[Category:Apps with no update check]]\n'
253 for appcat in app.Categories:
254 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
256 # We can't have underscores in the page name, even if they're in
257 # the package ID, because MediaWiki messes with them...
258 pagename = appid.replace('_', ' ')
260 # Drop a trailing newline, because mediawiki is going to drop it anyway
261 # and it we don't we'll think the page has changed when it hasn't...
262 if wikidata.endswith('\n'):
263 wikidata = wikidata[:-1]
265 generated_pages[pagename] = wikidata
267 # Make a redirect from the name to the ID too, unless there's
268 # already an existing page with the name and it isn't a redirect.
270 apppagename = app.Name.replace('_', ' ')
271 apppagename = apppagename.replace('{', '')
272 apppagename = apppagename.replace('}', ' ')
273 apppagename = apppagename.replace(':', ' ')
274 apppagename = apppagename.replace('[', ' ')
275 apppagename = apppagename.replace(']', ' ')
276 # Drop double spaces caused mostly by replacing ':' above
277 apppagename = apppagename.replace(' ', ' ')
278 for expagename in site.allpages(prefix=apppagename,
279 filterredir='nonredirects',
281 if expagename == apppagename:
283 # Another reason not to make the redirect page is if the app name
284 # is the same as it's ID, because that will overwrite the real page
285 # with an redirect to itself! (Although it seems like an odd
286 # scenario this happens a lot, e.g. where there is metadata but no
287 # builds or binaries to extract a name from.
288 if apppagename == pagename:
291 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
293 for tcat, genp in [(wikicat, generated_pages),
294 (wikiredircat, generated_redirects)]:
295 catpages = site.Pages['Category:' + tcat]
297 for page in catpages:
298 existingpages.append(page.name)
299 if page.name in genp:
300 pagetxt = page.edit()
301 if pagetxt != genp[page.name]:
302 logging.debug("Updating modified page " + page.name)
303 page.save(genp[page.name], summary='Auto-updated')
305 logging.debug("Page " + page.name + " is unchanged")
307 logging.warn("Deleting page " + page.name)
308 page.delete('No longer published')
309 for pagename, text in genp.items():
310 logging.debug("Checking " + pagename)
311 if pagename not in existingpages:
312 logging.debug("Creating page " + pagename)
314 newpage = site.Pages[pagename]
315 newpage.save(text, summary='Auto-created')
316 except Exception as e:
317 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
319 # Purge server cache to ensure counts are up to date
320 site.pages['Repository Maintenance'].purge()
323 def delete_disabled_builds(apps, apkcache, repodirs):
324 """Delete disabled build outputs.
326 :param apps: list of all applications, as per metadata.read_metadata
327 :param apkcache: current apk cache information
328 :param repodirs: the repo directories to process
330 for appid, app in apps.items():
331 for build in app['builds']:
332 if not build.disable:
334 apkfilename = common.get_release_filename(app, build)
335 iconfilename = "%s.%s.png" % (
338 for repodir in repodirs:
340 os.path.join(repodir, apkfilename),
341 os.path.join(repodir, apkfilename + '.asc'),
342 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
344 for density in all_screen_densities:
345 repo_dir = get_icon_dir(repodir, density)
346 files.append(os.path.join(repo_dir, iconfilename))
349 if os.path.exists(f):
350 logging.info("Deleting disabled build output " + f)
352 if apkfilename in apkcache:
353 del apkcache[apkfilename]
356 def resize_icon(iconpath, density):
358 if not os.path.isfile(iconpath):
363 fp = open(iconpath, 'rb')
365 size = dpi_to_px(density)
367 if any(length > size for length in im.size):
369 im.thumbnail((size, size), Image.ANTIALIAS)
370 logging.debug("%s was too large at %s - new size is %s" % (
371 iconpath, oldsize, im.size))
372 im.save(iconpath, "PNG")
374 except Exception as e:
375 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
382 def resize_all_icons(repodirs):
383 """Resize all icons that exceed the max size
385 :param repodirs: the repo directories to process
387 for repodir in repodirs:
388 for density in screen_densities:
389 icon_dir = get_icon_dir(repodir, density)
390 icon_glob = os.path.join(icon_dir, '*.png')
391 for iconpath in glob.glob(icon_glob):
392 resize_icon(iconpath, density)
396 """ Get the signing certificate of an apk. To get the same md5 has that
397 Android gets, we encode the .RSA certificate in a specific format and pass
398 it hex-encoded to the md5 digest algorithm.
400 :param apkpath: path to the apk
401 :returns: A string containing the md5 of the signature of the apk or None
402 if an error occurred.
405 # verify the jar signature is correct
406 if not common.verify_apk_signature(apkpath):
409 with zipfile.ZipFile(apkpath, 'r') as apk:
410 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
413 logging.error("Found no signing certificates on %s" % apkpath)
416 logging.error("Found multiple signing certificates on %s" % apkpath)
419 cert = apk.read(certs[0])
421 cert_encoded = common.get_certificate(cert)
423 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
426 def get_cache_file():
427 return os.path.join('tmp', 'apkcache')
432 Gather information about all the apk files in the repo directory,
433 using cached data if possible.
436 apkcachefile = get_cache_file()
437 if not options.clean and os.path.exists(apkcachefile):
438 with open(apkcachefile, 'rb') as cf:
439 apkcache = pickle.load(cf, encoding='utf-8')
440 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
448 def write_cache(apkcache):
449 apkcachefile = get_cache_file()
450 cache_path = os.path.dirname(apkcachefile)
451 if not os.path.exists(cache_path):
452 os.makedirs(cache_path)
453 apkcache["METADATA_VERSION"] = METADATA_VERSION
454 with open(apkcachefile, 'wb') as cf:
455 pickle.dump(apkcache, cf)
458 def get_icon_bytes(apkzip, iconsrc):
459 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
461 return apkzip.read(iconsrc)
463 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
466 def sha256sum(filename):
467 '''Calculate the sha256 of the given file'''
468 sha = hashlib.sha256()
469 with open(filename, 'rb') as f:
475 return sha.hexdigest()
478 def has_old_openssl(filename):
479 '''checks for known vulnerable openssl versions in the APK'''
481 # statically load this pattern
482 if not hasattr(has_old_openssl, "pattern"):
483 has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
485 with zipfile.ZipFile(filename) as zf:
486 for name in zf.namelist():
487 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
490 chunk = lib.read(4096)
493 m = has_old_openssl.pattern.search(chunk)
495 version = m.group(1).decode('ascii')
496 if version.startswith('1.0.1') and version[5] >= 'r' \
497 or version.startswith('1.0.2') and version[5] >= 'f':
498 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
500 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
506 def insert_obbs(repodir, apps, apks):
507 """Scans the .obb files in a given repo directory and adds them to the
508 relevant APK instances. OBB files have versionCodes like APK
509 files, and they are loosely associated. If there is an OBB file
510 present, then any APK with the same or higher versionCode will use
511 that OBB file. There are two OBB types: main and patch, each APK
512 can only have only have one of each.
514 https://developer.android.com/google/play/expansion-files.html
516 :param repodir: repo directory to scan
517 :param apps: list of current, valid apps
518 :param apks: current information on all APKs
522 def obbWarnDelete(f, msg):
523 logging.warning(msg + f)
524 if options.delete_unknown:
525 logging.error("Deleting unknown file: " + f)
529 java_Integer_MIN_VALUE = -pow(2, 31)
530 currentPackageNames = apps.keys()
531 for f in glob.glob(os.path.join(repodir, '*.obb')):
532 obbfile = os.path.basename(f)
533 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
534 chunks = obbfile.split('.')
535 if chunks[0] != 'main' and chunks[0] != 'patch':
536 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
538 if not re.match(r'^-?[0-9]+$', chunks[1]):
539 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
541 versionCode = int(chunks[1])
542 packagename = ".".join(chunks[2:-1])
544 highestVersionCode = java_Integer_MIN_VALUE
545 if packagename not in currentPackageNames:
546 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
549 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
550 highestVersionCode = apk['versionCode']
551 if versionCode > highestVersionCode:
552 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
553 + ') than any APK: ')
555 obbsha256 = sha256sum(f)
556 obbs.append((packagename, versionCode, obbfile, obbsha256))
559 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
560 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
561 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
562 apk['obbMainFile'] = obbfile
563 apk['obbMainFileSha256'] = obbsha256
564 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
565 apk['obbPatchFile'] = obbfile
566 apk['obbPatchFileSha256'] = obbsha256
567 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
571 def _get_localized_dict(app, locale):
572 '''get the dict to add localized store metadata to'''
573 if 'localized' not in app:
574 app['localized'] = collections.OrderedDict()
575 if locale not in app['localized']:
576 app['localized'][locale] = collections.OrderedDict()
577 return app['localized'][locale]
580 def _set_localized_text_entry(app, locale, key, f):
581 limit = config['char_limits'][key]
582 localized = _get_localized_dict(app, locale)
584 text = fp.read()[:limit]
586 localized[key] = text
589 def _set_author_entry(app, key, f):
590 limit = config['char_limits']['author']
592 text = fp.read()[:limit]
597 def copy_triple_t_store_metadata(apps):
598 """Include store metadata from the app's source repo
600 The Triple-T Gradle Play Publisher is a plugin that has a standard
601 file layout for all of the metadata and graphics that the Google
602 Play Store accepts. Since F-Droid has the git repo, it can just
603 pluck those files directly. This method reads any text files into
604 the app dict, then copies any graphics into the fdroid repo
607 This needs to be run before insert_localized_app_metadata() so that
608 the graphics files that are copied into the fdroid repo get
611 https://github.com/Triple-T/gradle-play-publisher#upload-images
612 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
616 if not os.path.isdir('build'):
617 return # nothing to do
619 for packageName, app in apps.items():
620 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
621 logging.debug('Triple-T Gradle Play Publisher: ' + d)
622 for root, dirs, files in os.walk(d):
623 segments = root.split('/')
624 locale = segments[-2]
626 if f == 'fulldescription':
627 _set_localized_text_entry(app, locale, 'description',
628 os.path.join(root, f))
630 elif f == 'shortdescription':
631 _set_localized_text_entry(app, locale, 'summary',
632 os.path.join(root, f))
635 _set_localized_text_entry(app, locale, 'name',
636 os.path.join(root, f))
639 _set_localized_text_entry(app, locale, 'video',
640 os.path.join(root, f))
642 elif f == 'whatsnew':
643 _set_localized_text_entry(app, segments[-1], 'whatsNew',
644 os.path.join(root, f))
646 elif f == 'contactEmail':
647 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
649 elif f == 'contactPhone':
650 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
652 elif f == 'contactWebsite':
653 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
656 base, extension = common.get_extension(f)
657 dirname = os.path.basename(root)
658 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
659 if segments[-2] == 'listing':
660 locale = segments[-3]
662 locale = segments[-2]
663 destdir = os.path.join('repo', packageName, locale)
664 os.makedirs(destdir, mode=0o755, exist_ok=True)
665 sourcefile = os.path.join(root, f)
666 destfile = os.path.join(destdir, dirname + '.' + extension)
667 logging.debug('copying ' + sourcefile + ' ' + destfile)
668 shutil.copy(sourcefile, destfile)
671 def insert_localized_app_metadata(apps):
672 """scans standard locations for graphics and localized text
674 Scans for localized description files, store graphics, and
675 screenshot PNG files in statically defined screenshots directory
676 and adds them to the app metadata. The screenshots and graphic
677 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
678 and must be in the following layout:
679 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
681 repo/packageName/locale/featureGraphic.png
682 repo/packageName/locale/phoneScreenshots/1.png
683 repo/packageName/locale/phoneScreenshots/2.png
685 The changelog files must be text files named with the versionCode
686 ending with ".txt" and must be in the following layout:
687 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
689 repo/packageName/locale/changelogs/12345.txt
691 This will scan the each app's source repo then the metadata/ dir
692 for these standard locations of changelog files. If it finds
693 them, they will be added to the dict of all packages, with the
694 versions in the metadata/ folder taking precendence over the what
695 is in the app's source repo.
697 Where "packageName" is the app's packageName and "locale" is the locale
698 of the graphics, e.g. what language they are in, using the IETF RFC5646
699 format (en-US, fr-CA, es-MX, etc).
701 This will also scan the app's git for a fastlane folder, and the
702 metadata/ folder and the apps' source repos for standard locations
703 of graphic and screenshot files. If it finds them, it will copy
704 them into the repo. The fastlane files follow this pattern:
705 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
709 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
710 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
712 for d in sorted(sourcedirs):
713 if not os.path.isdir(d):
715 for root, dirs, files in os.walk(d):
716 segments = root.split('/')
717 packageName = segments[1]
718 if packageName not in apps:
719 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
721 locale = segments[-1]
723 if f == 'full_description.txt':
724 _set_localized_text_entry(apps[packageName], locale, 'description',
725 os.path.join(root, f))
727 elif f == 'short_description.txt':
728 _set_localized_text_entry(apps[packageName], locale, 'summary',
729 os.path.join(root, f))
731 elif f == 'title.txt':
732 _set_localized_text_entry(apps[packageName], locale, 'name',
733 os.path.join(root, f))
735 elif f == 'video.txt':
736 _set_localized_text_entry(apps[packageName], locale, 'video',
737 os.path.join(root, f))
739 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
740 locale = segments[-2]
741 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
742 os.path.join(root, f))
745 base, extension = common.get_extension(f)
746 if locale == 'images':
747 locale = segments[-2]
748 destdir = os.path.join('repo', packageName, locale)
749 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
750 os.makedirs(destdir, mode=0o755, exist_ok=True)
751 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
752 shutil.copy(os.path.join(root, f), destdir)
754 if d in SCREENSHOT_DIRS:
755 for f in glob.glob(os.path.join(root, d, '*.*')):
756 _, extension = common.get_extension(f)
757 if extension in ALLOWED_EXTENSIONS:
758 screenshotdestdir = os.path.join(destdir, d)
759 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
760 logging.debug('copying ' + f + ' ' + screenshotdestdir)
761 shutil.copy(f, screenshotdestdir)
763 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
765 if not os.path.isdir(d):
767 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
768 if not os.path.isfile(f):
770 segments = f.split('/')
771 packageName = segments[1]
773 screenshotdir = segments[3]
774 filename = os.path.basename(f)
775 base, extension = common.get_extension(filename)
777 if packageName not in apps:
778 logging.warning('Found "%s" graphic without metadata for app "%s"!'
779 % (filename, packageName))
781 graphics = _get_localized_dict(apps[packageName], locale)
783 if extension not in ALLOWED_EXTENSIONS:
784 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
785 elif base in GRAPHIC_NAMES:
786 # there can only be zero or one of these per locale
787 graphics[base] = filename
788 elif screenshotdir in SCREENSHOT_DIRS:
789 # there can any number of these per locale
790 logging.debug('adding to ' + screenshotdir + ': ' + f)
791 if screenshotdir not in graphics:
792 graphics[screenshotdir] = []
793 graphics[screenshotdir].append(filename)
795 logging.warning('Unsupported graphics file found: ' + f)
798 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
799 """Scan a repo for all files with an extension except APK/OBB
801 :param apkcache: current cached info about all repo files
802 :param repodir: repo directory to scan
803 :param knownapks: list of all known files, as per metadata.read_metadata
804 :param use_date_from_file: use date from file (instead of current date)
805 for newly added files
810 repodir = repodir.encode('utf-8')
811 for name in os.listdir(repodir):
812 file_extension = common.get_file_extension(name)
813 if file_extension == 'apk' or file_extension == 'obb':
815 filename = os.path.join(repodir, name)
816 name_utf8 = name.decode('utf-8')
817 if filename.endswith(b'_src.tar.gz'):
818 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
820 if not common.is_repo_file(filename):
822 stat = os.stat(filename)
823 if stat.st_size == 0:
824 raise FDroidException(filename + ' is zero size!')
826 shasum = sha256sum(filename)
829 repo_file = apkcache[name]
830 # added time is cached as tuple but used here as datetime instance
831 if 'added' in repo_file:
832 a = repo_file['added']
833 if isinstance(a, datetime):
834 repo_file['added'] = a
836 repo_file['added'] = datetime(*a[:6])
837 if repo_file.get('hash') == shasum:
838 logging.debug("Reading " + name_utf8 + " from cache")
841 logging.debug("Ignoring stale cache data for " + name)
844 logging.debug("Processing " + name_utf8)
845 repo_file = collections.OrderedDict()
846 # TODO rename apkname globally to something more generic
847 repo_file['name'] = name_utf8
848 repo_file['apkName'] = name_utf8
849 repo_file['hash'] = shasum
850 repo_file['hashType'] = 'sha256'
851 repo_file['versionCode'] = 0
852 repo_file['versionName'] = shasum
853 # the static ID is the SHA256 unless it is set in the metadata
854 repo_file['packageName'] = shasum
856 n = name_utf8.rsplit('_', maxsplit=1)
859 versionCode = n[1].split('.')[0]
860 if re.match('^-?[0-9]+$', versionCode) \
861 and common.is_valid_package_name(n[0]):
862 repo_file['packageName'] = packageName
863 repo_file['versionCode'] = int(versionCode)
864 srcfilename = name + b'_src.tar.gz'
865 if os.path.exists(os.path.join(repodir, srcfilename)):
866 repo_file['srcname'] = srcfilename.decode('utf-8')
867 repo_file['size'] = stat.st_size
869 apkcache[name] = repo_file
872 if use_date_from_file:
873 timestamp = stat.st_ctime
874 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
876 default_date_param = None
878 # Record in knownapks, getting the added date at the same time..
879 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
880 default_date=default_date_param)
882 repo_file['added'] = added
884 repo_files.append(repo_file)
886 return repo_files, cachechanged
889 def scan_apk_aapt(apk, apkfile):
890 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
891 if p.returncode != 0:
892 if options.delete_unknown:
893 if os.path.exists(apkfile):
894 logging.error("Failed to get apk information, deleting " + apkfile)
897 logging.error("Could not find {0} to remove it".format(apkfile))
899 logging.error("Failed to get apk information, skipping " + apkfile)
900 raise BuildException("Invalid APK")
901 for line in p.output.splitlines():
902 if line.startswith("package:"):
904 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
905 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
906 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
907 except Exception as e:
908 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
909 elif line.startswith("application:"):
910 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
911 # Keep path to non-dpi icon in case we need it
912 match = re.match(APK_ICON_PAT_NODPI, line)
914 apk['icons_src']['-1'] = match.group(1)
915 elif line.startswith("launchable-activity:"):
916 # Only use launchable-activity as fallback to application
918 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
919 if '-1' not in apk['icons_src']:
920 match = re.match(APK_ICON_PAT_NODPI, line)
922 apk['icons_src']['-1'] = match.group(1)
923 elif line.startswith("application-icon-"):
924 match = re.match(APK_ICON_PAT, line)
926 density = match.group(1)
927 path = match.group(2)
928 apk['icons_src'][density] = path
929 elif line.startswith("sdkVersion:"):
930 m = re.match(APK_SDK_VERSION_PAT, line)
932 logging.error(line.replace('sdkVersion:', '')
933 + ' is not a valid minSdkVersion!')
935 apk['minSdkVersion'] = m.group(1)
936 # if target not set, default to min
937 if 'targetSdkVersion' not in apk:
938 apk['targetSdkVersion'] = m.group(1)
939 elif line.startswith("targetSdkVersion:"):
940 m = re.match(APK_SDK_VERSION_PAT, line)
942 logging.error(line.replace('targetSdkVersion:', '')
943 + ' is not a valid targetSdkVersion!')
945 apk['targetSdkVersion'] = m.group(1)
946 elif line.startswith("maxSdkVersion:"):
947 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
948 elif line.startswith("native-code:"):
949 apk['nativecode'] = []
950 for arch in line[13:].split(' '):
951 apk['nativecode'].append(arch[1:-1])
952 elif line.startswith('uses-permission:'):
953 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
954 if perm_match['maxSdkVersion']:
955 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
956 permission = UsesPermission(
958 perm_match['maxSdkVersion']
961 apk['uses-permission'].append(permission)
962 elif line.startswith('uses-permission-sdk-23:'):
963 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
964 if perm_match['maxSdkVersion']:
965 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
966 permission_sdk_23 = UsesPermissionSdk23(
968 perm_match['maxSdkVersion']
971 apk['uses-permission-sdk-23'].append(permission_sdk_23)
973 elif line.startswith('uses-feature:'):
974 feature = re.match(APK_FEATURE_PAT, line).group(1)
975 # Filter out this, it's only added with the latest SDK tools and
976 # causes problems for lots of apps.
977 if feature != "android.hardware.screen.portrait" \
978 and feature != "android.hardware.screen.landscape":
979 if feature.startswith("android.feature."):
980 feature = feature[16:]
981 apk['features'].add(feature)
984 def scan_apk_androguard(apk, apkfile):
986 from androguard.core.bytecodes.apk import APK
987 apkobject = APK(apkfile)
988 if apkobject.is_valid_APK():
989 arsc = apkobject.get_android_resources()
991 if options.delete_unknown:
992 if os.path.exists(apkfile):
993 logging.error("Failed to get apk information, deleting " + apkfile)
996 logging.error("Could not find {0} to remove it".format(apkfile))
998 logging.error("Failed to get apk information, skipping " + apkfile)
999 raise BuildException("Invaild APK")
1001 raise FDroidException("androguard library is not installed and aapt not present")
1002 except FileNotFoundError:
1003 logging.error("Could not open apk file for analysis")
1004 raise BuildException("Invalid APK")
1006 apk['packageName'] = apkobject.get_package()
1007 apk['versionCode'] = int(apkobject.get_androidversion_code())
1008 apk['versionName'] = apkobject.get_androidversion_name()
1009 if apk['versionName'][0] == "@":
1010 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1011 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1012 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1013 apk['name'] = apkobject.get_app_name()
1015 if apkobject.get_max_sdk_version() is not None:
1016 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1017 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1018 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1020 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1021 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1023 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1025 for file in apkobject.get_files():
1026 d_re = density_re.match(file)
1028 folder = d_re.group(1).split('-')
1030 resolution = folder[1]
1033 density = screen_resolutions[resolution]
1034 apk['icons_src'][density] = d_re.group(0)
1036 if apk['icons_src'].get('-1') is None:
1037 apk['icons_src']['-1'] = apk['icons_src']['160']
1039 arch_re = re.compile("^lib/(.*)/.*$")
1040 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1042 apk['nativecode'] = []
1043 apk['nativecode'].extend(sorted(list(arch)))
1045 xml = apkobject.get_android_manifest_xml()
1047 for item in xml.getElementsByTagName('uses-permission'):
1048 name = str(item.getAttribute("android:name"))
1049 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1050 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1051 permission = UsesPermission(
1055 apk['uses-permission'].append(permission)
1057 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1058 name = str(item.getAttribute("android:name"))
1059 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1060 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1061 permission_sdk_23 = UsesPermissionSdk23(
1065 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1067 for item in xml.getElementsByTagName('uses-feature'):
1068 feature = str(item.getAttribute("android:name"))
1069 if feature != "android.hardware.screen.portrait" \
1070 and feature != "android.hardware.screen.landscape":
1071 if feature.startswith("android.feature."):
1072 feature = feature[16:]
1073 apk['features'].append(feature)
1076 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1077 """Scan the apk with the given filename in the given repo directory.
1079 This also extracts the icons.
1081 :param apkcache: current apk cache information
1082 :param apkfilename: the filename of the apk to scan
1083 :param repodir: repo directory to scan
1084 :param knownapks: known apks info
1085 :param use_date_from_apk: use date from APK (instead of current date)
1086 for newly added APKs
1087 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1088 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1091 if ' ' in apkfilename:
1092 logging.critical("Spaces in filenames are not allowed.")
1093 return True, None, False
1095 apkfile = os.path.join(repodir, apkfilename)
1096 shasum = sha256sum(apkfile)
1098 cachechanged = False
1100 if apkfilename in apkcache:
1101 apk = apkcache[apkfilename]
1102 if apk.get('hash') == shasum:
1103 logging.debug("Reading " + apkfilename + " from cache")
1106 logging.debug("Ignoring stale cache data for " + apkfilename)
1109 logging.debug("Processing " + apkfilename)
1111 apk['apkName'] = apkfilename
1112 apk['hash'] = shasum
1113 apk['hashType'] = 'sha256'
1114 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1115 if os.path.exists(os.path.join(repodir, srcfilename)):
1116 apk['srcname'] = srcfilename
1117 apk['size'] = os.path.getsize(apkfile)
1118 apk['uses-permission'] = []
1119 apk['uses-permission-sdk-23'] = []
1120 apk['features'] = []
1121 apk['icons_src'] = {}
1123 apk['antiFeatures'] = set()
1124 if has_old_openssl(apkfile):
1125 apk['antiFeatures'].add('KnownVuln')
1128 if SdkToolsPopen(['aapt', 'version'], output=False):
1129 scan_apk_aapt(apk, apkfile)
1131 scan_apk_androguard(apk, apkfile)
1132 except BuildException:
1133 return True, None, False
1135 if 'minSdkVersion' not in apk:
1136 logging.warn("No SDK version information found in {0}".format(apkfile))
1137 apk['minSdkVersion'] = 1
1139 # Check for debuggable apks...
1140 if common.isApkAndDebuggable(apkfile):
1141 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1143 # Get the signature (or md5 of, to be precise)...
1144 logging.debug('Getting signature of {0}'.format(apkfile))
1145 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1147 logging.critical("Failed to get apk signature")
1148 return True, None, False
1150 apkzip = zipfile.ZipFile(apkfile, 'r')
1152 # if an APK has files newer than the system time, suggest updating
1153 # the system clock. This is useful for offline systems, used for
1154 # signing, which do not have another source of clock sync info. It
1155 # has to be more than 24 hours newer because ZIP/APK files do not
1156 # store timezone info
1157 manifest = apkzip.getinfo('AndroidManifest.xml')
1158 if manifest.date_time[1] == 0: # month can't be zero
1159 logging.debug('AndroidManifest.xml has no date')
1161 dt_obj = datetime(*manifest.date_time)
1162 checkdt = dt_obj - timedelta(1)
1163 if datetime.today() < checkdt:
1164 logging.warn('System clock is older than manifest in: '
1166 + '\nSet clock to that time using:\n'
1167 + 'sudo date -s "' + str(dt_obj) + '"')
1169 iconfilename = "%s.%s.png" % (
1173 # Extract the icon file...
1174 empty_densities = []
1175 for density in screen_densities:
1176 if density not in apk['icons_src']:
1177 empty_densities.append(density)
1179 iconsrc = apk['icons_src'][density]
1180 icon_dir = get_icon_dir(repodir, density)
1181 icondest = os.path.join(icon_dir, iconfilename)
1184 with open(icondest, 'wb') as f:
1185 f.write(get_icon_bytes(apkzip, iconsrc))
1186 apk['icons'][density] = iconfilename
1187 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1188 logging.warning("Error retrieving icon file: %s" % (icondest))
1189 del apk['icons_src'][density]
1190 empty_densities.append(density)
1192 if '-1' in apk['icons_src']:
1193 iconsrc = apk['icons_src']['-1']
1194 iconpath = os.path.join(
1195 get_icon_dir(repodir, '0'), iconfilename)
1196 with open(iconpath, 'wb') as f:
1197 f.write(get_icon_bytes(apkzip, iconsrc))
1199 im = Image.open(iconpath)
1200 dpi = px_to_dpi(im.size[0])
1201 for density in screen_densities:
1202 if density in apk['icons']:
1204 if density == screen_densities[-1] or dpi >= int(density):
1205 apk['icons'][density] = iconfilename
1206 shutil.move(iconpath,
1207 os.path.join(get_icon_dir(repodir, density), iconfilename))
1208 empty_densities.remove(density)
1210 except Exception as e:
1211 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1214 apk['icon'] = iconfilename
1218 # First try resizing down to not lose quality
1220 for density in screen_densities:
1221 if density not in empty_densities:
1222 last_density = density
1224 if last_density is None:
1226 logging.debug("Density %s not available, resizing down from %s"
1227 % (density, last_density))
1229 last_iconpath = os.path.join(
1230 get_icon_dir(repodir, last_density), iconfilename)
1231 iconpath = os.path.join(
1232 get_icon_dir(repodir, density), iconfilename)
1235 fp = open(last_iconpath, 'rb')
1238 size = dpi_to_px(density)
1240 im.thumbnail((size, size), Image.ANTIALIAS)
1241 im.save(iconpath, "PNG")
1242 empty_densities.remove(density)
1243 except Exception as e:
1244 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1249 # Then just copy from the highest resolution available
1251 for density in reversed(screen_densities):
1252 if density not in empty_densities:
1253 last_density = density
1255 if last_density is None:
1257 logging.debug("Density %s not available, copying from lower density %s"
1258 % (density, last_density))
1261 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1262 os.path.join(get_icon_dir(repodir, density), iconfilename))
1264 empty_densities.remove(density)
1266 for density in screen_densities:
1267 icon_dir = get_icon_dir(repodir, density)
1268 icondest = os.path.join(icon_dir, iconfilename)
1269 resize_icon(icondest, density)
1271 # Copy from icons-mdpi to icons since mdpi is the baseline density
1272 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1273 if os.path.isfile(baseline):
1274 apk['icons']['0'] = iconfilename
1275 shutil.copyfile(baseline,
1276 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1278 if use_date_from_apk and manifest.date_time[1] != 0:
1279 default_date_param = datetime(*manifest.date_time)
1281 default_date_param = None
1283 # Record in known apks, getting the added date at the same time..
1284 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1285 default_date=default_date_param)
1287 apk['added'] = added
1289 apkcache[apkfilename] = apk
1292 return False, apk, cachechanged
1295 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1296 """Scan the apks in the given repo directory.
1298 This also extracts the icons.
1300 :param apkcache: current apk cache information
1301 :param repodir: repo directory to scan
1302 :param knownapks: known apks info
1303 :param use_date_from_apk: use date from APK (instead of current date)
1304 for newly added APKs
1305 :returns: (apks, cachechanged) where apks is a list of apk information,
1306 and cachechanged is True if the apkcache got changed.
1309 cachechanged = False
1311 for icon_dir in get_all_icon_dirs(repodir):
1312 if os.path.exists(icon_dir):
1314 shutil.rmtree(icon_dir)
1315 os.makedirs(icon_dir)
1317 os.makedirs(icon_dir)
1320 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1321 apkfilename = apkfile[len(repodir) + 1:]
1322 (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1327 return apks, cachechanged
1330 def apply_info_from_latest_apk(apps, apks):
1332 Some information from the apks needs to be applied up to the application level.
1333 When doing this, we use the info from the most recent version's apk.
1334 We deal with figuring out when the app was added and last updated at the same time.
1336 for appid, app in apps.items():
1337 bestver = UNSET_VERSION_CODE
1339 if apk['packageName'] == appid:
1340 if apk['versionCode'] > bestver:
1341 bestver = apk['versionCode']
1345 if not app.added or apk['added'] < app.added:
1346 app.added = apk['added']
1347 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1348 app.lastUpdated = apk['added']
1351 logging.debug("Don't know when " + appid + " was added")
1352 if not app.lastUpdated:
1353 logging.debug("Don't know when " + appid + " was last updated")
1355 if bestver == UNSET_VERSION_CODE:
1357 if app.Name is None:
1358 app.Name = app.AutoName or appid
1360 logging.debug("Application " + appid + " has no packages")
1362 if app.Name is None:
1363 app.Name = bestapk['name']
1364 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1365 if app.CurrentVersionCode is None:
1366 app.CurrentVersionCode = str(bestver)
1369 def make_categories_txt(repodir, categories):
1370 '''Write a category list in the repo to allow quick access'''
1372 for cat in sorted(categories):
1373 catdata += cat + '\n'
1374 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1378 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1380 for appid, app in apps.items():
1382 if app.ArchivePolicy:
1383 keepversions = int(app.ArchivePolicy[:-9])
1385 keepversions = defaultkeepversions
1387 def filter_apk_list_sorted(apk_list):
1389 for apk in apk_list:
1390 if apk['packageName'] == appid:
1393 # Sort the apk list by version code. First is highest/newest.
1394 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1396 def move_file(from_dir, to_dir, filename, ignore_missing):
1397 from_path = os.path.join(from_dir, filename)
1398 if ignore_missing and not os.path.exists(from_path):
1400 to_path = os.path.join(to_dir, filename)
1401 shutil.move(from_path, to_path)
1403 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1404 .format(appid, len(apks), keepversions, len(archapks)))
1406 if len(apks) > keepversions:
1407 apklist = filter_apk_list_sorted(apks)
1408 # Move back the ones we don't want.
1409 for apk in apklist[keepversions:]:
1410 logging.info("Moving " + apk['apkName'] + " to archive")
1411 move_file(repodir, archivedir, apk['apkName'], False)
1412 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1413 for density in all_screen_densities:
1414 repo_icon_dir = get_icon_dir(repodir, density)
1415 archive_icon_dir = get_icon_dir(archivedir, density)
1416 if density not in apk['icons']:
1418 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1419 if 'srcname' in apk:
1420 move_file(repodir, archivedir, apk['srcname'], False)
1421 archapks.append(apk)
1423 elif len(apks) < keepversions and len(archapks) > 0:
1424 required = keepversions - len(apks)
1425 archapklist = filter_apk_list_sorted(archapks)
1426 # Move forward the ones we want again.
1427 for apk in archapklist[:required]:
1428 logging.info("Moving " + apk['apkName'] + " from archive")
1429 move_file(archivedir, repodir, apk['apkName'], False)
1430 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1431 for density in all_screen_densities:
1432 repo_icon_dir = get_icon_dir(repodir, density)
1433 archive_icon_dir = get_icon_dir(archivedir, density)
1434 if density not in apk['icons']:
1436 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1437 if 'srcname' in apk:
1438 move_file(archivedir, repodir, apk['srcname'], False)
1439 archapks.remove(apk)
1443 def add_apks_to_per_app_repos(repodir, apks):
1444 apks_per_app = dict()
1446 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1447 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1448 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1449 apks_per_app[apk['packageName']] = apk
1451 if not os.path.exists(apk['per_app_icons']):
1452 logging.info('Adding new repo for only ' + apk['packageName'])
1453 os.makedirs(apk['per_app_icons'])
1455 apkpath = os.path.join(repodir, apk['apkName'])
1456 shutil.copy(apkpath, apk['per_app_repo'])
1457 apksigpath = apkpath + '.sig'
1458 if os.path.exists(apksigpath):
1459 shutil.copy(apksigpath, apk['per_app_repo'])
1460 apkascpath = apkpath + '.asc'
1461 if os.path.exists(apkascpath):
1462 shutil.copy(apkascpath, apk['per_app_repo'])
1471 global config, options
1473 # Parse command line...
1474 parser = ArgumentParser()
1475 common.setup_global_opts(parser)
1476 parser.add_argument("--create-key", action="store_true", default=False,
1477 help="Create a repo signing key in a keystore")
1478 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1479 help="Create skeleton metadata files that are missing")
1480 parser.add_argument("--delete-unknown", action="store_true", default=False,
1481 help="Delete APKs and/or OBBs without metadata from the repo")
1482 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1483 help="Report on build data status")
1484 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1485 help="Interactively ask about things that need updating.")
1486 parser.add_argument("-I", "--icons", action="store_true", default=False,
1487 help="Resize all the icons exceeding the max pixel size and exit")
1488 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1489 help="Specify editor to use in interactive mode. Default " +
1490 "is /etc/alternatives/editor")
1491 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1492 help="Update the wiki")
1493 parser.add_argument("--pretty", action="store_true", default=False,
1494 help="Produce human-readable index.xml")
1495 parser.add_argument("--clean", action="store_true", default=False,
1496 help="Clean update - don't uses caches, reprocess all apks")
1497 parser.add_argument("--nosign", action="store_true", default=False,
1498 help="When configured for signed indexes, create only unsigned indexes at this stage")
1499 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1500 help="Use date from apk instead of current time for newly added apks")
1501 metadata.add_metadata_arguments(parser)
1502 options = parser.parse_args()
1503 metadata.warnings_action = options.W
1505 config = common.read_config(options)
1507 if not ('jarsigner' in config and 'keytool' in config):
1508 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1511 if config['archive_older'] != 0:
1512 repodirs.append('archive')
1513 if not os.path.exists('archive'):
1517 resize_all_icons(repodirs)
1520 # check that icons exist now, rather than fail at the end of `fdroid update`
1521 for k in ['repo_icon', 'archive_icon']:
1523 if not os.path.exists(config[k]):
1524 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1527 # if the user asks to create a keystore, do it now, reusing whatever it can
1528 if options.create_key:
1529 if os.path.exists(config['keystore']):
1530 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1531 logging.critical("\t'" + config['keystore'] + "'")
1534 if 'repo_keyalias' not in config:
1535 config['repo_keyalias'] = socket.getfqdn()
1536 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1537 if 'keydname' not in config:
1538 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1539 common.write_to_config(config, 'keydname', config['keydname'])
1540 if 'keystore' not in config:
1541 config['keystore'] = common.default_config['keystore']
1542 common.write_to_config(config, 'keystore', config['keystore'])
1544 password = common.genpassword()
1545 if 'keystorepass' not in config:
1546 config['keystorepass'] = password
1547 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1548 if 'keypass' not in config:
1549 config['keypass'] = password
1550 common.write_to_config(config, 'keypass', config['keypass'])
1551 common.genkeystore(config)
1554 apps = metadata.read_metadata()
1556 # Generate a list of categories...
1558 for app in apps.values():
1559 categories.update(app.Categories)
1561 # Read known apks data (will be updated and written back when we've finished)
1562 knownapks = common.KnownApks()
1565 apkcache = get_cache()
1567 # Delete builds for disabled apps
1568 delete_disabled_builds(apps, apkcache, repodirs)
1570 # Scan all apks in the main repo
1571 apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1573 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1574 options.use_date_from_apk)
1575 cachechanged = cachechanged or fcachechanged
1577 # Generate warnings for apk's with no metadata (or create skeleton
1578 # metadata files, if requested on the command line)
1581 if apk['packageName'] not in apps:
1582 if options.create_metadata:
1583 if 'name' not in apk:
1584 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1586 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1587 f.write("License:Unknown\n")
1588 f.write("Web Site:\n")
1589 f.write("Source Code:\n")
1590 f.write("Issue Tracker:\n")
1591 f.write("Changelog:\n")
1592 f.write("Summary:" + apk['name'] + "\n")
1593 f.write("Description:\n")
1594 f.write(apk['name'] + "\n")
1596 f.write("Name:" + apk['name'] + "\n")
1598 logging.info("Generated skeleton metadata for " + apk['packageName'])
1601 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1602 if options.delete_unknown:
1603 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1604 rmf = os.path.join(repodirs[0], apk['apkName'])
1605 if not os.path.exists(rmf):
1606 logging.error("Could not find {0} to remove it".format(rmf))
1610 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1612 # update the metadata with the newly created ones included
1614 apps = metadata.read_metadata()
1616 copy_triple_t_store_metadata(apps)
1617 insert_obbs(repodirs[0], apps, apks)
1618 insert_localized_app_metadata(apps)
1620 # Scan the archive repo for apks as well
1621 if len(repodirs) > 1:
1622 archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1628 # Apply information from latest apks to the application and update dates
1629 apply_info_from_latest_apk(apps, apks + archapks)
1631 # Sort the app list by name, then the web site doesn't have to by default.
1632 # (we had to wait until we'd scanned the apks to do this, because mostly the
1633 # name comes from there!)
1634 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1636 # APKs are placed into multiple repos based on the app package, providing
1637 # per-app subscription feeds for nightly builds and things like it
1638 if config['per_app_repos']:
1639 add_apks_to_per_app_repos(repodirs[0], apks)
1640 for appid, app in apps.items():
1641 repodir = os.path.join(appid, 'fdroid', 'repo')
1643 appdict[appid] = app
1644 if os.path.isdir(repodir):
1645 index.make(appdict, [appid], apks, repodir, False)
1647 logging.info('Skipping index generation for ' + appid)
1650 if len(repodirs) > 1:
1651 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1653 # Make the index for the main repo...
1654 index.make(apps, sortedids, apks, repodirs[0], False)
1655 make_categories_txt(repodirs[0], categories)
1657 # If there's an archive repo, make the index for it. We already scanned it
1659 if len(repodirs) > 1:
1660 index.make(apps, sortedids, archapks, repodirs[1], True)
1662 git_remote = config.get('binary_transparency_remote')
1663 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1664 btlog.make_binary_transparency_log(repodirs)
1666 if config['update_stats']:
1667 # Update known apks info...
1668 knownapks.writeifchanged()
1670 # Generate latest apps data for widget
1671 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1673 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1675 appid = line.rstrip()
1676 data += appid + "\t"
1678 data += app.Name + "\t"
1679 if app.icon is not None:
1680 data += app.icon + "\t"
1681 data += app.License + "\n"
1682 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1686 write_cache(apkcache)
1688 # Update the wiki...
1690 update_wiki(apps, sortedids, apks + archapks)
1692 logging.info("Finished.")
1695 if __name__ == "__main__":