3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
76 all_screen_densities = ['0'] + screen_densities
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 def dpi_to_px(density):
88 return (int(density) * 48) / 160
92 return (int(px) * 160) / 48
95 def get_icon_dir(repodir, density):
97 return os.path.join(repodir, "icons")
98 return os.path.join(repodir, "icons-%s" % density)
101 def get_icon_dirs(repodir):
102 for density in screen_densities:
103 yield get_icon_dir(repodir, density)
106 def get_all_icon_dirs(repodir):
107 for density in all_screen_densities:
108 yield get_icon_dir(repodir, density)
111 def update_wiki(apps, sortedids, apks):
114 :param apps: fully populated list of all applications
115 :param apks: all apks, except...
117 logging.info("Updating wiki")
119 wikiredircat = 'App Redirects'
121 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122 path=config['wiki_path'])
123 site.login(config['wiki_user'], config['wiki_password'])
125 generated_redirects = {}
127 for appid in sortedids:
128 app = metadata.App(apps[appid])
132 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
134 for af in app.AntiFeatures:
135 wikidata += '{{AntiFeature|' + af + '}}\n'
140 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
143 app.added.strftime('%Y-%m-%d') if app.added else '',
144 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
159 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
161 wikidata += app.Summary
162 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
164 wikidata += "=Description=\n"
165 wikidata += metadata.description_wiki(app.Description) + "\n"
167 wikidata += "=Maintainer Notes=\n"
168 if app.MaintainerNotes:
169 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
172 # Get a list of all packages for this application...
174 gotcurrentver = False
178 if apk['packageName'] == appid:
179 if str(apk['versionCode']) == app.CurrentVersionCode:
182 # Include ones we can't build, as a special case...
183 for build in app.builds:
185 if build.versionCode == app.CurrentVersionCode:
187 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188 apklist.append({'versionCode': int(build.versionCode),
189 'versionName': build.versionName,
190 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
195 if apk['versionCode'] == int(build.versionCode):
200 apklist.append({'versionCode': int(build.versionCode),
201 'versionName': build.versionName,
202 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
204 if app.CurrentVersionCode == '0':
206 # Sort with most recent first...
207 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
209 wikidata += "=Versions=\n"
210 if len(apklist) == 0:
211 wikidata += "We currently have no versions of this app available."
212 elif not gotcurrentver:
213 wikidata += "We don't have the current version of this app."
215 wikidata += "We have the current version of this app."
216 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218 if len(app.NoSourceSince) > 0:
219 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220 if len(app.CurrentVersion) > 0:
221 wikidata += "The current (recommended) version is " + app.CurrentVersion
222 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
225 wikidata += "==" + apk['versionName'] + "==\n"
227 if 'buildproblem' in apk:
228 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
231 wikidata += "This version is built and signed by "
233 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
235 wikidata += "the original developer.\n\n"
236 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
238 wikidata += '\n[[Category:' + wikicat + ']]\n'
239 if len(app.NoSourceSince) > 0:
240 wikidata += '\n[[Category:Apps missing source code]]\n'
241 if validapks == 0 and not app.Disabled:
242 wikidata += '\n[[Category:Apps with no packages]]\n'
243 if cantupdate and not app.Disabled:
244 wikidata += "\n[[Category:Apps we cannot update]]\n"
245 if buildfails and not app.Disabled:
246 wikidata += "\n[[Category:Apps with failing builds]]\n"
247 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248 wikidata += '\n[[Category:Apps to Update]]\n'
250 wikidata += '\n[[Category:Apps that are disabled]]\n'
251 if app.UpdateCheckMode == 'None' and not app.Disabled:
252 wikidata += '\n[[Category:Apps with no update check]]\n'
253 for appcat in app.Categories:
254 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
256 # We can't have underscores in the page name, even if they're in
257 # the package ID, because MediaWiki messes with them...
258 pagename = appid.replace('_', ' ')
260 # Drop a trailing newline, because mediawiki is going to drop it anyway
261 # and it we don't we'll think the page has changed when it hasn't...
262 if wikidata.endswith('\n'):
263 wikidata = wikidata[:-1]
265 generated_pages[pagename] = wikidata
267 # Make a redirect from the name to the ID too, unless there's
268 # already an existing page with the name and it isn't a redirect.
270 apppagename = app.Name.replace('_', ' ')
271 apppagename = apppagename.replace('{', '')
272 apppagename = apppagename.replace('}', ' ')
273 apppagename = apppagename.replace(':', ' ')
274 apppagename = apppagename.replace('[', ' ')
275 apppagename = apppagename.replace(']', ' ')
276 # Drop double spaces caused mostly by replacing ':' above
277 apppagename = apppagename.replace(' ', ' ')
278 for expagename in site.allpages(prefix=apppagename,
279 filterredir='nonredirects',
281 if expagename == apppagename:
283 # Another reason not to make the redirect page is if the app name
284 # is the same as it's ID, because that will overwrite the real page
285 # with an redirect to itself! (Although it seems like an odd
286 # scenario this happens a lot, e.g. where there is metadata but no
287 # builds or binaries to extract a name from.
288 if apppagename == pagename:
291 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
293 for tcat, genp in [(wikicat, generated_pages),
294 (wikiredircat, generated_redirects)]:
295 catpages = site.Pages['Category:' + tcat]
297 for page in catpages:
298 existingpages.append(page.name)
299 if page.name in genp:
300 pagetxt = page.edit()
301 if pagetxt != genp[page.name]:
302 logging.debug("Updating modified page " + page.name)
303 page.save(genp[page.name], summary='Auto-updated')
305 logging.debug("Page " + page.name + " is unchanged")
307 logging.warn("Deleting page " + page.name)
308 page.delete('No longer published')
309 for pagename, text in genp.items():
310 logging.debug("Checking " + pagename)
311 if pagename not in existingpages:
312 logging.debug("Creating page " + pagename)
314 newpage = site.Pages[pagename]
315 newpage.save(text, summary='Auto-created')
316 except Exception as e:
317 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
319 # Purge server cache to ensure counts are up to date
320 site.pages['Repository Maintenance'].purge()
323 def delete_disabled_builds(apps, apkcache, repodirs):
324 """Delete disabled build outputs.
326 :param apps: list of all applications, as per metadata.read_metadata
327 :param apkcache: current apk cache information
328 :param repodirs: the repo directories to process
330 for appid, app in apps.items():
331 for build in app['builds']:
332 if not build.disable:
334 apkfilename = common.get_release_filename(app, build)
335 iconfilename = "%s.%s.png" % (
338 for repodir in repodirs:
340 os.path.join(repodir, apkfilename),
341 os.path.join(repodir, apkfilename + '.asc'),
342 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
344 for density in all_screen_densities:
345 repo_dir = get_icon_dir(repodir, density)
346 files.append(os.path.join(repo_dir, iconfilename))
349 if os.path.exists(f):
350 logging.info("Deleting disabled build output " + f)
352 if apkfilename in apkcache:
353 del apkcache[apkfilename]
356 def resize_icon(iconpath, density):
358 if not os.path.isfile(iconpath):
363 fp = open(iconpath, 'rb')
365 size = dpi_to_px(density)
367 if any(length > size for length in im.size):
369 im.thumbnail((size, size), Image.ANTIALIAS)
370 logging.debug("%s was too large at %s - new size is %s" % (
371 iconpath, oldsize, im.size))
372 im.save(iconpath, "PNG")
374 except Exception as e:
375 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
382 def resize_all_icons(repodirs):
383 """Resize all icons that exceed the max size
385 :param repodirs: the repo directories to process
387 for repodir in repodirs:
388 for density in screen_densities:
389 icon_dir = get_icon_dir(repodir, density)
390 icon_glob = os.path.join(icon_dir, '*.png')
391 for iconpath in glob.glob(icon_glob):
392 resize_icon(iconpath, density)
396 """ Get the signing certificate of an apk. To get the same md5 has that
397 Android gets, we encode the .RSA certificate in a specific format and pass
398 it hex-encoded to the md5 digest algorithm.
400 :param apkpath: path to the apk
401 :returns: A string containing the md5 of the signature of the apk or None
402 if an error occurred.
405 # verify the jar signature is correct
406 if not common.verify_apk_signature(apkpath):
409 with zipfile.ZipFile(apkpath, 'r') as apk:
410 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
413 logging.error("Found no signing certificates on %s" % apkpath)
416 logging.error("Found multiple signing certificates on %s" % apkpath)
419 cert = apk.read(certs[0])
421 cert_encoded = common.get_certificate(cert)
423 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
426 def get_cache_file():
427 return os.path.join('tmp', 'apkcache')
432 Gather information about all the apk files in the repo directory,
433 using cached data if possible.
436 apkcachefile = get_cache_file()
437 if not options.clean and os.path.exists(apkcachefile):
438 with open(apkcachefile, 'rb') as cf:
439 apkcache = pickle.load(cf, encoding='utf-8')
440 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
448 def write_cache(apkcache):
449 apkcachefile = get_cache_file()
450 cache_path = os.path.dirname(apkcachefile)
451 if not os.path.exists(cache_path):
452 os.makedirs(cache_path)
453 apkcache["METADATA_VERSION"] = METADATA_VERSION
454 with open(apkcachefile, 'wb') as cf:
455 pickle.dump(apkcache, cf)
458 def get_icon_bytes(apkzip, iconsrc):
459 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
461 return apkzip.read(iconsrc)
463 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
466 def sha256sum(filename):
467 '''Calculate the sha256 of the given file'''
468 sha = hashlib.sha256()
469 with open(filename, 'rb') as f:
475 return sha.hexdigest()
478 def has_old_openssl(filename):
479 '''checks for known vulnerable openssl versions in the APK'''
481 # statically load this pattern
482 if not hasattr(has_old_openssl, "pattern"):
483 has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
485 with zipfile.ZipFile(filename) as zf:
486 for name in zf.namelist():
487 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
490 chunk = lib.read(4096)
493 m = has_old_openssl.pattern.search(chunk)
495 version = m.group(1).decode('ascii')
496 if version.startswith('1.0.1') and version[5] >= 'r' \
497 or version.startswith('1.0.2') and version[5] >= 'f':
498 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
500 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
506 def insert_obbs(repodir, apps, apks):
507 """Scans the .obb files in a given repo directory and adds them to the
508 relevant APK instances. OBB files have versionCodes like APK
509 files, and they are loosely associated. If there is an OBB file
510 present, then any APK with the same or higher versionCode will use
511 that OBB file. There are two OBB types: main and patch, each APK
512 can only have only have one of each.
514 https://developer.android.com/google/play/expansion-files.html
516 :param repodir: repo directory to scan
517 :param apps: list of current, valid apps
518 :param apks: current information on all APKs
522 def obbWarnDelete(f, msg):
523 logging.warning(msg + f)
524 if options.delete_unknown:
525 logging.error("Deleting unknown file: " + f)
529 java_Integer_MIN_VALUE = -pow(2, 31)
530 currentPackageNames = apps.keys()
531 for f in glob.glob(os.path.join(repodir, '*.obb')):
532 obbfile = os.path.basename(f)
533 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
534 chunks = obbfile.split('.')
535 if chunks[0] != 'main' and chunks[0] != 'patch':
536 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
538 if not re.match(r'^-?[0-9]+$', chunks[1]):
539 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
541 versionCode = int(chunks[1])
542 packagename = ".".join(chunks[2:-1])
544 highestVersionCode = java_Integer_MIN_VALUE
545 if packagename not in currentPackageNames:
546 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
549 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
550 highestVersionCode = apk['versionCode']
551 if versionCode > highestVersionCode:
552 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
553 + ') than any APK: ')
555 obbsha256 = sha256sum(f)
556 obbs.append((packagename, versionCode, obbfile, obbsha256))
559 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
560 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
561 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
562 apk['obbMainFile'] = obbfile
563 apk['obbMainFileSha256'] = obbsha256
564 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
565 apk['obbPatchFile'] = obbfile
566 apk['obbPatchFileSha256'] = obbsha256
567 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
571 def _get_localized_dict(app, locale):
572 '''get the dict to add localized store metadata to'''
573 if 'localized' not in app:
574 app['localized'] = collections.OrderedDict()
575 if locale not in app['localized']:
576 app['localized'][locale] = collections.OrderedDict()
577 return app['localized'][locale]
580 def _set_localized_text_entry(app, locale, key, f):
581 limit = config['char_limits'][key]
582 localized = _get_localized_dict(app, locale)
584 text = fp.read()[:limit]
586 localized[key] = text
589 def _set_author_entry(app, key, f):
590 limit = config['char_limits']['author']
592 text = fp.read()[:limit]
597 def copy_triple_t_store_metadata(apps):
598 """Include store metadata from the app's source repo
600 The Triple-T Gradle Play Publisher is a plugin that has a standard
601 file layout for all of the metadata and graphics that the Google
602 Play Store accepts. Since F-Droid has the git repo, it can just
603 pluck those files directly. This method reads any text files into
604 the app dict, then copies any graphics into the fdroid repo
607 This needs to be run before insert_localized_app_metadata() so that
608 the graphics files that are copied into the fdroid repo get
611 https://github.com/Triple-T/gradle-play-publisher#upload-images
612 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
616 if not os.path.isdir('build'):
617 return # nothing to do
619 for packageName, app in apps.items():
620 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
621 logging.debug('Triple-T Gradle Play Publisher: ' + d)
622 for root, dirs, files in os.walk(d):
623 segments = root.split('/')
624 locale = segments[-2]
626 if f == 'fulldescription':
627 _set_localized_text_entry(app, locale, 'description',
628 os.path.join(root, f))
630 elif f == 'shortdescription':
631 _set_localized_text_entry(app, locale, 'summary',
632 os.path.join(root, f))
635 _set_localized_text_entry(app, locale, 'name',
636 os.path.join(root, f))
639 _set_localized_text_entry(app, locale, 'video',
640 os.path.join(root, f))
642 elif f == 'whatsnew':
643 _set_localized_text_entry(app, segments[-1], 'whatsNew',
644 os.path.join(root, f))
646 elif f == 'contactEmail':
647 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
649 elif f == 'contactPhone':
650 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
652 elif f == 'contactWebsite':
653 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
656 base, extension = common.get_extension(f)
657 dirname = os.path.basename(root)
658 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
659 if segments[-2] == 'listing':
660 locale = segments[-3]
662 locale = segments[-2]
663 destdir = os.path.join('repo', packageName, locale)
664 os.makedirs(destdir, mode=0o755, exist_ok=True)
665 sourcefile = os.path.join(root, f)
666 destfile = os.path.join(destdir, dirname + '.' + extension)
667 logging.debug('copying ' + sourcefile + ' ' + destfile)
668 shutil.copy(sourcefile, destfile)
671 def insert_localized_app_metadata(apps):
672 """scans standard locations for graphics and localized text
674 Scans for localized description files, store graphics, and
675 screenshot PNG files in statically defined screenshots directory
676 and adds them to the app metadata. The screenshots and graphic
677 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
678 and must be in the following layout:
679 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
681 repo/packageName/locale/featureGraphic.png
682 repo/packageName/locale/phoneScreenshots/1.png
683 repo/packageName/locale/phoneScreenshots/2.png
685 The changelog files must be text files named with the versionCode
686 ending with ".txt" and must be in the following layout:
687 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
689 repo/packageName/locale/changelogs/12345.txt
691 This will scan the each app's source repo then the metadata/ dir
692 for these standard locations of changelog files. If it finds
693 them, they will be added to the dict of all packages, with the
694 versions in the metadata/ folder taking precendence over the what
695 is in the app's source repo.
697 Where "packageName" is the app's packageName and "locale" is the locale
698 of the graphics, e.g. what language they are in, using the IETF RFC5646
699 format (en-US, fr-CA, es-MX, etc).
701 This will also scan the app's git for a fastlane folder, and the
702 metadata/ folder and the apps' source repos for standard locations
703 of graphic and screenshot files. If it finds them, it will copy
704 them into the repo. The fastlane files follow this pattern:
705 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
709 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
710 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
712 for d in sorted(sourcedirs):
713 if not os.path.isdir(d):
715 for root, dirs, files in os.walk(d):
716 segments = root.split('/')
717 packageName = segments[1]
718 if packageName not in apps:
719 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
721 locale = segments[-1]
723 if f == 'full_description.txt':
724 _set_localized_text_entry(apps[packageName], locale, 'description',
725 os.path.join(root, f))
727 elif f == 'short_description.txt':
728 _set_localized_text_entry(apps[packageName], locale, 'summary',
729 os.path.join(root, f))
731 elif f == 'title.txt':
732 _set_localized_text_entry(apps[packageName], locale, 'name',
733 os.path.join(root, f))
735 elif f == 'video.txt':
736 _set_localized_text_entry(apps[packageName], locale, 'video',
737 os.path.join(root, f))
739 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
740 locale = segments[-2]
741 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
742 os.path.join(root, f))
745 base, extension = common.get_extension(f)
746 if locale == 'images':
747 locale = segments[-2]
748 destdir = os.path.join('repo', packageName, locale)
749 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
750 os.makedirs(destdir, mode=0o755, exist_ok=True)
751 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
752 shutil.copy(os.path.join(root, f), destdir)
754 if d in SCREENSHOT_DIRS:
755 for f in glob.glob(os.path.join(root, d, '*.*')):
756 _, extension = common.get_extension(f)
757 if extension in ALLOWED_EXTENSIONS:
758 screenshotdestdir = os.path.join(destdir, d)
759 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
760 logging.debug('copying ' + f + ' ' + screenshotdestdir)
761 shutil.copy(f, screenshotdestdir)
763 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
765 if not os.path.isdir(d):
767 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
768 if not os.path.isfile(f):
770 segments = f.split('/')
771 packageName = segments[1]
773 screenshotdir = segments[3]
774 filename = os.path.basename(f)
775 base, extension = common.get_extension(filename)
777 if packageName not in apps:
778 logging.warning('Found "%s" graphic without metadata for app "%s"!'
779 % (filename, packageName))
781 graphics = _get_localized_dict(apps[packageName], locale)
783 if extension not in ALLOWED_EXTENSIONS:
784 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
785 elif base in GRAPHIC_NAMES:
786 # there can only be zero or one of these per locale
787 graphics[base] = filename
788 elif screenshotdir in SCREENSHOT_DIRS:
789 # there can any number of these per locale
790 logging.debug('adding to ' + screenshotdir + ': ' + f)
791 if screenshotdir not in graphics:
792 graphics[screenshotdir] = []
793 graphics[screenshotdir].append(filename)
795 logging.warning('Unsupported graphics file found: ' + f)
798 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
799 """Scan a repo for all files with an extension except APK/OBB
801 :param apkcache: current cached info about all repo files
802 :param repodir: repo directory to scan
803 :param knownapks: list of all known files, as per metadata.read_metadata
804 :param use_date_from_file: use date from file (instead of current date)
805 for newly added files
810 repodir = repodir.encode('utf-8')
811 for name in os.listdir(repodir):
812 file_extension = common.get_file_extension(name)
813 if file_extension == 'apk' or file_extension == 'obb':
815 filename = os.path.join(repodir, name)
816 name_utf8 = name.decode('utf-8')
817 if filename.endswith(b'_src.tar.gz'):
818 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
820 if not common.is_repo_file(filename):
822 stat = os.stat(filename)
823 if stat.st_size == 0:
824 raise FDroidException(filename + ' is zero size!')
826 shasum = sha256sum(filename)
829 repo_file = apkcache[name]
830 # added time is cached as tuple but used here as datetime instance
831 if 'added' in repo_file:
832 a = repo_file['added']
833 if isinstance(a, datetime):
834 repo_file['added'] = a
836 repo_file['added'] = datetime(*a[:6])
837 if repo_file.get('hash') == shasum:
838 logging.debug("Reading " + name_utf8 + " from cache")
841 logging.debug("Ignoring stale cache data for " + name)
844 logging.debug("Processing " + name_utf8)
845 repo_file = collections.OrderedDict()
846 # TODO rename apkname globally to something more generic
847 repo_file['name'] = name_utf8
848 repo_file['apkName'] = name_utf8
849 repo_file['hash'] = shasum
850 repo_file['hashType'] = 'sha256'
851 repo_file['versionCode'] = 0
852 repo_file['versionName'] = shasum
853 # the static ID is the SHA256 unless it is set in the metadata
854 repo_file['packageName'] = shasum
856 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
858 repo_file['packageName'] = m.group(1)
859 repo_file['versionCode'] = int(m.group(2))
860 srcfilename = name + b'_src.tar.gz'
861 if os.path.exists(os.path.join(repodir, srcfilename)):
862 repo_file['srcname'] = srcfilename.decode('utf-8')
863 repo_file['size'] = stat.st_size
865 apkcache[name] = repo_file
868 if use_date_from_file:
869 timestamp = stat.st_ctime
870 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
872 default_date_param = None
874 # Record in knownapks, getting the added date at the same time..
875 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
876 default_date=default_date_param)
878 repo_file['added'] = added
880 repo_files.append(repo_file)
882 return repo_files, cachechanged
885 def scan_apk_aapt(apk, apkfile):
886 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
887 if p.returncode != 0:
888 if options.delete_unknown:
889 if os.path.exists(apkfile):
890 logging.error("Failed to get apk information, deleting " + apkfile)
893 logging.error("Could not find {0} to remove it".format(apkfile))
895 logging.error("Failed to get apk information, skipping " + apkfile)
896 raise BuildException("Invalid APK")
897 for line in p.output.splitlines():
898 if line.startswith("package:"):
900 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
901 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
902 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
903 except Exception as e:
904 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
905 elif line.startswith("application:"):
906 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
907 # Keep path to non-dpi icon in case we need it
908 match = re.match(APK_ICON_PAT_NODPI, line)
910 apk['icons_src']['-1'] = match.group(1)
911 elif line.startswith("launchable-activity:"):
912 # Only use launchable-activity as fallback to application
914 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
915 if '-1' not in apk['icons_src']:
916 match = re.match(APK_ICON_PAT_NODPI, line)
918 apk['icons_src']['-1'] = match.group(1)
919 elif line.startswith("application-icon-"):
920 match = re.match(APK_ICON_PAT, line)
922 density = match.group(1)
923 path = match.group(2)
924 apk['icons_src'][density] = path
925 elif line.startswith("sdkVersion:"):
926 m = re.match(APK_SDK_VERSION_PAT, line)
928 logging.error(line.replace('sdkVersion:', '')
929 + ' is not a valid minSdkVersion!')
931 apk['minSdkVersion'] = m.group(1)
932 # if target not set, default to min
933 if 'targetSdkVersion' not in apk:
934 apk['targetSdkVersion'] = m.group(1)
935 elif line.startswith("targetSdkVersion:"):
936 m = re.match(APK_SDK_VERSION_PAT, line)
938 logging.error(line.replace('targetSdkVersion:', '')
939 + ' is not a valid targetSdkVersion!')
941 apk['targetSdkVersion'] = m.group(1)
942 elif line.startswith("maxSdkVersion:"):
943 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
944 elif line.startswith("native-code:"):
945 apk['nativecode'] = []
946 for arch in line[13:].split(' '):
947 apk['nativecode'].append(arch[1:-1])
948 elif line.startswith('uses-permission:'):
949 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
950 if perm_match['maxSdkVersion']:
951 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
952 permission = UsesPermission(
954 perm_match['maxSdkVersion']
957 apk['uses-permission'].append(permission)
958 elif line.startswith('uses-permission-sdk-23:'):
959 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
960 if perm_match['maxSdkVersion']:
961 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
962 permission_sdk_23 = UsesPermissionSdk23(
964 perm_match['maxSdkVersion']
967 apk['uses-permission-sdk-23'].append(permission_sdk_23)
969 elif line.startswith('uses-feature:'):
970 feature = re.match(APK_FEATURE_PAT, line).group(1)
971 # Filter out this, it's only added with the latest SDK tools and
972 # causes problems for lots of apps.
973 if feature != "android.hardware.screen.portrait" \
974 and feature != "android.hardware.screen.landscape":
975 if feature.startswith("android.feature."):
976 feature = feature[16:]
977 apk['features'].add(feature)
980 def scan_apk_androguard(apk, apkfile):
982 from androguard.core.bytecodes.apk import APK
983 apkobject = APK(apkfile)
984 if apkobject.is_valid_APK():
985 arsc = apkobject.get_android_resources()
987 if options.delete_unknown:
988 if os.path.exists(apkfile):
989 logging.error("Failed to get apk information, deleting " + apkfile)
992 logging.error("Could not find {0} to remove it".format(apkfile))
994 logging.error("Failed to get apk information, skipping " + apkfile)
995 raise BuildException("Invaild APK")
997 raise FDroidException("androguard library is not installed and aapt not present")
998 except FileNotFoundError:
999 logging.error("Could not open apk file for analysis")
1000 raise BuildException("Invalid APK")
1002 apk['packageName'] = apkobject.get_package()
1003 apk['versionCode'] = int(apkobject.get_androidversion_code())
1004 apk['versionName'] = apkobject.get_androidversion_name()
1005 if apk['versionName'][0] == "@":
1006 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1007 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1008 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1009 apk['name'] = apkobject.get_app_name()
1011 if apkobject.get_max_sdk_version() is not None:
1012 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1013 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1014 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1016 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1017 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1019 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1021 for file in apkobject.get_files():
1022 d_re = density_re.match(file)
1024 folder = d_re.group(1).split('-')
1026 resolution = folder[1]
1029 density = screen_resolutions[resolution]
1030 apk['icons_src'][density] = d_re.group(0)
1032 if apk['icons_src'].get('-1') is None:
1033 apk['icons_src']['-1'] = apk['icons_src']['160']
1035 arch_re = re.compile("^lib/(.*)/.*$")
1036 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1038 apk['nativecode'] = []
1039 apk['nativecode'].extend(sorted(list(arch)))
1041 xml = apkobject.get_android_manifest_xml()
1043 for item in xml.getElementsByTagName('uses-permission'):
1044 name = str(item.getAttribute("android:name"))
1045 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1046 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1047 permission = UsesPermission(
1051 apk['uses-permission'].append(permission)
1053 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1054 name = str(item.getAttribute("android:name"))
1055 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1056 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1057 permission_sdk_23 = UsesPermissionSdk23(
1061 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1063 for item in xml.getElementsByTagName('uses-feature'):
1064 feature = str(item.getAttribute("android:name"))
1065 if feature != "android.hardware.screen.portrait" \
1066 and feature != "android.hardware.screen.landscape":
1067 if feature.startswith("android.feature."):
1068 feature = feature[16:]
1069 apk['features'].append(feature)
1072 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1073 """Scan the apk with the given filename in the given repo directory.
1075 This also extracts the icons.
1077 :param apkcache: current apk cache information
1078 :param apkfilename: the filename of the apk to scan
1079 :param repodir: repo directory to scan
1080 :param knownapks: known apks info
1081 :param use_date_from_apk: use date from APK (instead of current date)
1082 for newly added APKs
1083 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1084 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1087 if ' ' in apkfilename:
1088 if options.rename_apks:
1089 newfilename = apkfilename.replace(' ', '_')
1090 os.rename(os.path.join(repodir, apkfilename),
1091 os.path.join(repodir, newfilename))
1092 apkfilename = newfilename
1094 logging.critical("Spaces in filenames are not allowed.")
1095 return True, None, False
1097 apkfile = os.path.join(repodir, apkfilename)
1098 shasum = sha256sum(apkfile)
1100 cachechanged = False
1102 if apkfilename in apkcache:
1103 apk = apkcache[apkfilename]
1104 if apk.get('hash') == shasum:
1105 logging.debug("Reading " + apkfilename + " from cache")
1108 logging.debug("Ignoring stale cache data for " + apkfilename)
1111 logging.debug("Processing " + apkfilename)
1113 apk['hash'] = shasum
1114 apk['hashType'] = 'sha256'
1115 apk['uses-permission'] = []
1116 apk['uses-permission-sdk-23'] = []
1117 apk['features'] = []
1118 apk['icons_src'] = {}
1120 apk['antiFeatures'] = set()
1121 if has_old_openssl(apkfile):
1122 apk['antiFeatures'].add('KnownVuln')
1125 if SdkToolsPopen(['aapt', 'version'], output=False):
1126 scan_apk_aapt(apk, apkfile)
1128 scan_apk_androguard(apk, apkfile)
1129 except BuildException:
1130 return True, None, False
1132 if 'minSdkVersion' not in apk:
1133 logging.warn("No SDK version information found in {0}".format(apkfile))
1134 apk['minSdkVersion'] = 1
1136 # Check for debuggable apks...
1137 if common.isApkAndDebuggable(apkfile):
1138 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1140 # Get the signature (or md5 of, to be precise)...
1141 logging.debug('Getting signature of {0}'.format(apkfile))
1142 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1144 logging.critical("Failed to get apk signature")
1145 return True, None, False
1147 if options.rename_apks:
1148 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1149 std_short_name = os.path.join(repodir, n)
1150 if apkfile != std_short_name:
1151 if os.path.exists(std_short_name):
1152 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1153 if apkfile != std_long_name:
1154 if os.path.exists(std_long_name):
1155 dupdir = os.path.join('duplicates', repodir)
1156 if not os.path.isdir(dupdir):
1157 os.makedirs(dupdir, exist_ok=True)
1158 dupfile = os.path.join('duplicates', std_long_name)
1159 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1160 os.rename(apkfile, dupfile)
1161 return True, None, False
1163 os.rename(apkfile, std_long_name)
1164 apkfile = std_long_name
1166 os.rename(apkfile, std_short_name)
1167 apkfile = std_short_name
1168 apkfilename = apkfile[len(repodir) + 1:]
1170 apk['apkName'] = apkfilename
1171 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1172 if os.path.exists(os.path.join(repodir, srcfilename)):
1173 apk['srcname'] = srcfilename
1174 apk['size'] = os.path.getsize(apkfile)
1176 apkzip = zipfile.ZipFile(apkfile, 'r')
1178 # if an APK has files newer than the system time, suggest updating
1179 # the system clock. This is useful for offline systems, used for
1180 # signing, which do not have another source of clock sync info. It
1181 # has to be more than 24 hours newer because ZIP/APK files do not
1182 # store timezone info
1183 manifest = apkzip.getinfo('AndroidManifest.xml')
1184 if manifest.date_time[1] == 0: # month can't be zero
1185 logging.debug('AndroidManifest.xml has no date')
1187 dt_obj = datetime(*manifest.date_time)
1188 checkdt = dt_obj - timedelta(1)
1189 if datetime.today() < checkdt:
1190 logging.warn('System clock is older than manifest in: '
1192 + '\nSet clock to that time using:\n'
1193 + 'sudo date -s "' + str(dt_obj) + '"')
1195 iconfilename = "%s.%s.png" % (
1199 # Extract the icon file...
1200 empty_densities = []
1201 for density in screen_densities:
1202 if density not in apk['icons_src']:
1203 empty_densities.append(density)
1205 iconsrc = apk['icons_src'][density]
1206 icon_dir = get_icon_dir(repodir, density)
1207 icondest = os.path.join(icon_dir, iconfilename)
1210 with open(icondest, 'wb') as f:
1211 f.write(get_icon_bytes(apkzip, iconsrc))
1212 apk['icons'][density] = iconfilename
1213 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1214 logging.warning("Error retrieving icon file: %s" % (icondest))
1215 del apk['icons_src'][density]
1216 empty_densities.append(density)
1218 if '-1' in apk['icons_src']:
1219 iconsrc = apk['icons_src']['-1']
1220 iconpath = os.path.join(
1221 get_icon_dir(repodir, '0'), iconfilename)
1222 with open(iconpath, 'wb') as f:
1223 f.write(get_icon_bytes(apkzip, iconsrc))
1225 im = Image.open(iconpath)
1226 dpi = px_to_dpi(im.size[0])
1227 for density in screen_densities:
1228 if density in apk['icons']:
1230 if density == screen_densities[-1] or dpi >= int(density):
1231 apk['icons'][density] = iconfilename
1232 shutil.move(iconpath,
1233 os.path.join(get_icon_dir(repodir, density), iconfilename))
1234 empty_densities.remove(density)
1236 except Exception as e:
1237 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1240 apk['icon'] = iconfilename
1244 # First try resizing down to not lose quality
1246 for density in screen_densities:
1247 if density not in empty_densities:
1248 last_density = density
1250 if last_density is None:
1252 logging.debug("Density %s not available, resizing down from %s"
1253 % (density, last_density))
1255 last_iconpath = os.path.join(
1256 get_icon_dir(repodir, last_density), iconfilename)
1257 iconpath = os.path.join(
1258 get_icon_dir(repodir, density), iconfilename)
1261 fp = open(last_iconpath, 'rb')
1264 size = dpi_to_px(density)
1266 im.thumbnail((size, size), Image.ANTIALIAS)
1267 im.save(iconpath, "PNG")
1268 empty_densities.remove(density)
1269 except Exception as e:
1270 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1275 # Then just copy from the highest resolution available
1277 for density in reversed(screen_densities):
1278 if density not in empty_densities:
1279 last_density = density
1281 if last_density is None:
1283 logging.debug("Density %s not available, copying from lower density %s"
1284 % (density, last_density))
1287 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1288 os.path.join(get_icon_dir(repodir, density), iconfilename))
1290 empty_densities.remove(density)
1292 for density in screen_densities:
1293 icon_dir = get_icon_dir(repodir, density)
1294 icondest = os.path.join(icon_dir, iconfilename)
1295 resize_icon(icondest, density)
1297 # Copy from icons-mdpi to icons since mdpi is the baseline density
1298 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1299 if os.path.isfile(baseline):
1300 apk['icons']['0'] = iconfilename
1301 shutil.copyfile(baseline,
1302 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1304 if use_date_from_apk and manifest.date_time[1] != 0:
1305 default_date_param = datetime(*manifest.date_time)
1307 default_date_param = None
1309 # Record in known apks, getting the added date at the same time..
1310 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1311 default_date=default_date_param)
1313 apk['added'] = added
1315 apkcache[apkfilename] = apk
1318 return False, apk, cachechanged
1321 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1322 """Scan the apks in the given repo directory.
1324 This also extracts the icons.
1326 :param apkcache: current apk cache information
1327 :param repodir: repo directory to scan
1328 :param knownapks: known apks info
1329 :param use_date_from_apk: use date from APK (instead of current date)
1330 for newly added APKs
1331 :returns: (apks, cachechanged) where apks is a list of apk information,
1332 and cachechanged is True if the apkcache got changed.
1335 cachechanged = False
1337 for icon_dir in get_all_icon_dirs(repodir):
1338 if os.path.exists(icon_dir):
1340 shutil.rmtree(icon_dir)
1341 os.makedirs(icon_dir)
1343 os.makedirs(icon_dir)
1346 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1347 apkfilename = apkfile[len(repodir) + 1:]
1348 (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1353 return apks, cachechanged
1356 def apply_info_from_latest_apk(apps, apks):
1358 Some information from the apks needs to be applied up to the application level.
1359 When doing this, we use the info from the most recent version's apk.
1360 We deal with figuring out when the app was added and last updated at the same time.
1362 for appid, app in apps.items():
1363 bestver = UNSET_VERSION_CODE
1365 if apk['packageName'] == appid:
1366 if apk['versionCode'] > bestver:
1367 bestver = apk['versionCode']
1371 if not app.added or apk['added'] < app.added:
1372 app.added = apk['added']
1373 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1374 app.lastUpdated = apk['added']
1377 logging.debug("Don't know when " + appid + " was added")
1378 if not app.lastUpdated:
1379 logging.debug("Don't know when " + appid + " was last updated")
1381 if bestver == UNSET_VERSION_CODE:
1383 if app.Name is None:
1384 app.Name = app.AutoName or appid
1386 logging.debug("Application " + appid + " has no packages")
1388 if app.Name is None:
1389 app.Name = bestapk['name']
1390 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1391 if app.CurrentVersionCode is None:
1392 app.CurrentVersionCode = str(bestver)
1395 def make_categories_txt(repodir, categories):
1396 '''Write a category list in the repo to allow quick access'''
1398 for cat in sorted(categories):
1399 catdata += cat + '\n'
1400 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1404 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1406 for appid, app in apps.items():
1408 if app.ArchivePolicy:
1409 keepversions = int(app.ArchivePolicy[:-9])
1411 keepversions = defaultkeepversions
1413 def filter_apk_list_sorted(apk_list):
1415 for apk in apk_list:
1416 if apk['packageName'] == appid:
1419 # Sort the apk list by version code. First is highest/newest.
1420 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1422 def move_file(from_dir, to_dir, filename, ignore_missing):
1423 from_path = os.path.join(from_dir, filename)
1424 if ignore_missing and not os.path.exists(from_path):
1426 to_path = os.path.join(to_dir, filename)
1427 shutil.move(from_path, to_path)
1429 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1430 .format(appid, len(apks), keepversions, len(archapks)))
1432 if len(apks) > keepversions:
1433 apklist = filter_apk_list_sorted(apks)
1434 # Move back the ones we don't want.
1435 for apk in apklist[keepversions:]:
1436 logging.info("Moving " + apk['apkName'] + " to archive")
1437 move_file(repodir, archivedir, apk['apkName'], False)
1438 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1439 for density in all_screen_densities:
1440 repo_icon_dir = get_icon_dir(repodir, density)
1441 archive_icon_dir = get_icon_dir(archivedir, density)
1442 if density not in apk['icons']:
1444 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1445 if 'srcname' in apk:
1446 move_file(repodir, archivedir, apk['srcname'], False)
1447 archapks.append(apk)
1449 elif len(apks) < keepversions and len(archapks) > 0:
1450 required = keepversions - len(apks)
1451 archapklist = filter_apk_list_sorted(archapks)
1452 # Move forward the ones we want again.
1453 for apk in archapklist[:required]:
1454 logging.info("Moving " + apk['apkName'] + " from archive")
1455 move_file(archivedir, repodir, apk['apkName'], False)
1456 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1457 for density in all_screen_densities:
1458 repo_icon_dir = get_icon_dir(repodir, density)
1459 archive_icon_dir = get_icon_dir(archivedir, density)
1460 if density not in apk['icons']:
1462 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1463 if 'srcname' in apk:
1464 move_file(archivedir, repodir, apk['srcname'], False)
1465 archapks.remove(apk)
1469 def add_apks_to_per_app_repos(repodir, apks):
1470 apks_per_app = dict()
1472 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1473 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1474 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1475 apks_per_app[apk['packageName']] = apk
1477 if not os.path.exists(apk['per_app_icons']):
1478 logging.info('Adding new repo for only ' + apk['packageName'])
1479 os.makedirs(apk['per_app_icons'])
1481 apkpath = os.path.join(repodir, apk['apkName'])
1482 shutil.copy(apkpath, apk['per_app_repo'])
1483 apksigpath = apkpath + '.sig'
1484 if os.path.exists(apksigpath):
1485 shutil.copy(apksigpath, apk['per_app_repo'])
1486 apkascpath = apkpath + '.asc'
1487 if os.path.exists(apkascpath):
1488 shutil.copy(apkascpath, apk['per_app_repo'])
1497 global config, options
1499 # Parse command line...
1500 parser = ArgumentParser()
1501 common.setup_global_opts(parser)
1502 parser.add_argument("--create-key", action="store_true", default=False,
1503 help="Create a repo signing key in a keystore")
1504 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1505 help="Create skeleton metadata files that are missing")
1506 parser.add_argument("--delete-unknown", action="store_true", default=False,
1507 help="Delete APKs and/or OBBs without metadata from the repo")
1508 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1509 help="Report on build data status")
1510 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1511 help="Interactively ask about things that need updating.")
1512 parser.add_argument("-I", "--icons", action="store_true", default=False,
1513 help="Resize all the icons exceeding the max pixel size and exit")
1514 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1515 help="Specify editor to use in interactive mode. Default " +
1516 "is /etc/alternatives/editor")
1517 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1518 help="Update the wiki")
1519 parser.add_argument("--pretty", action="store_true", default=False,
1520 help="Produce human-readable index.xml")
1521 parser.add_argument("--clean", action="store_true", default=False,
1522 help="Clean update - don't uses caches, reprocess all apks")
1523 parser.add_argument("--nosign", action="store_true", default=False,
1524 help="When configured for signed indexes, create only unsigned indexes at this stage")
1525 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1526 help="Use date from apk instead of current time for newly added apks")
1527 parser.add_argument("--rename-apks", action="store_true", default=False,
1528 help="Rename APK files that do not match package.name_123.apk")
1529 metadata.add_metadata_arguments(parser)
1530 options = parser.parse_args()
1531 metadata.warnings_action = options.W
1533 config = common.read_config(options)
1535 if not ('jarsigner' in config and 'keytool' in config):
1536 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1539 if config['archive_older'] != 0:
1540 repodirs.append('archive')
1541 if not os.path.exists('archive'):
1545 resize_all_icons(repodirs)
1548 if options.rename_apks:
1549 options.clean = True
1551 # check that icons exist now, rather than fail at the end of `fdroid update`
1552 for k in ['repo_icon', 'archive_icon']:
1554 if not os.path.exists(config[k]):
1555 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1558 # if the user asks to create a keystore, do it now, reusing whatever it can
1559 if options.create_key:
1560 if os.path.exists(config['keystore']):
1561 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1562 logging.critical("\t'" + config['keystore'] + "'")
1565 if 'repo_keyalias' not in config:
1566 config['repo_keyalias'] = socket.getfqdn()
1567 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1568 if 'keydname' not in config:
1569 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1570 common.write_to_config(config, 'keydname', config['keydname'])
1571 if 'keystore' not in config:
1572 config['keystore'] = common.default_config['keystore']
1573 common.write_to_config(config, 'keystore', config['keystore'])
1575 password = common.genpassword()
1576 if 'keystorepass' not in config:
1577 config['keystorepass'] = password
1578 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1579 if 'keypass' not in config:
1580 config['keypass'] = password
1581 common.write_to_config(config, 'keypass', config['keypass'])
1582 common.genkeystore(config)
1585 apps = metadata.read_metadata()
1587 # Generate a list of categories...
1589 for app in apps.values():
1590 categories.update(app.Categories)
1592 # Read known apks data (will be updated and written back when we've finished)
1593 knownapks = common.KnownApks()
1596 apkcache = get_cache()
1598 # Delete builds for disabled apps
1599 delete_disabled_builds(apps, apkcache, repodirs)
1601 # Scan all apks in the main repo
1602 apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1604 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1605 options.use_date_from_apk)
1606 cachechanged = cachechanged or fcachechanged
1608 # Generate warnings for apk's with no metadata (or create skeleton
1609 # metadata files, if requested on the command line)
1612 if apk['packageName'] not in apps:
1613 if options.create_metadata:
1614 if 'name' not in apk:
1615 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1617 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1618 f.write("License:Unknown\n")
1619 f.write("Web Site:\n")
1620 f.write("Source Code:\n")
1621 f.write("Issue Tracker:\n")
1622 f.write("Changelog:\n")
1623 f.write("Summary:" + apk['name'] + "\n")
1624 f.write("Description:\n")
1625 f.write(apk['name'] + "\n")
1627 f.write("Name:" + apk['name'] + "\n")
1629 logging.info("Generated skeleton metadata for " + apk['packageName'])
1632 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1633 if options.delete_unknown:
1634 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1635 rmf = os.path.join(repodirs[0], apk['apkName'])
1636 if not os.path.exists(rmf):
1637 logging.error("Could not find {0} to remove it".format(rmf))
1641 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1643 # update the metadata with the newly created ones included
1645 apps = metadata.read_metadata()
1647 copy_triple_t_store_metadata(apps)
1648 insert_obbs(repodirs[0], apps, apks)
1649 insert_localized_app_metadata(apps)
1651 # Scan the archive repo for apks as well
1652 if len(repodirs) > 1:
1653 archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1659 # Apply information from latest apks to the application and update dates
1660 apply_info_from_latest_apk(apps, apks + archapks)
1662 # Sort the app list by name, then the web site doesn't have to by default.
1663 # (we had to wait until we'd scanned the apks to do this, because mostly the
1664 # name comes from there!)
1665 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1667 # APKs are placed into multiple repos based on the app package, providing
1668 # per-app subscription feeds for nightly builds and things like it
1669 if config['per_app_repos']:
1670 add_apks_to_per_app_repos(repodirs[0], apks)
1671 for appid, app in apps.items():
1672 repodir = os.path.join(appid, 'fdroid', 'repo')
1674 appdict[appid] = app
1675 if os.path.isdir(repodir):
1676 index.make(appdict, [appid], apks, repodir, False)
1678 logging.info('Skipping index generation for ' + appid)
1681 if len(repodirs) > 1:
1682 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1684 # Make the index for the main repo...
1685 index.make(apps, sortedids, apks, repodirs[0], False)
1686 make_categories_txt(repodirs[0], categories)
1688 # If there's an archive repo, make the index for it. We already scanned it
1690 if len(repodirs) > 1:
1691 index.make(apps, sortedids, archapks, repodirs[1], True)
1693 git_remote = config.get('binary_transparency_remote')
1694 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1695 btlog.make_binary_transparency_log(repodirs)
1697 if config['update_stats']:
1698 # Update known apks info...
1699 knownapks.writeifchanged()
1701 # Generate latest apps data for widget
1702 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1704 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1706 appid = line.rstrip()
1707 data += appid + "\t"
1709 data += app.Name + "\t"
1710 if app.icon is not None:
1711 data += app.icon + "\t"
1712 data += app.License + "\n"
1713 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1717 write_cache(apkcache)
1719 # Update the wiki...
1721 update_wiki(apps, sortedids, apks + archapks)
1723 logging.info("Finished.")
1726 if __name__ == "__main__":