3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
76 all_screen_densities = ['0'] + screen_densities
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
87 def dpi_to_px(density):
88 return (int(density) * 48) / 160
92 return (int(px) * 160) / 48
95 def get_icon_dir(repodir, density):
97 return os.path.join(repodir, "icons")
98 return os.path.join(repodir, "icons-%s" % density)
101 def get_icon_dirs(repodir):
102 for density in screen_densities:
103 yield get_icon_dir(repodir, density)
106 def get_all_icon_dirs(repodir):
107 for density in all_screen_densities:
108 yield get_icon_dir(repodir, density)
111 def update_wiki(apps, sortedids, apks):
114 :param apps: fully populated list of all applications
115 :param apks: all apks, except...
117 logging.info("Updating wiki")
119 wikiredircat = 'App Redirects'
121 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122 path=config['wiki_path'])
123 site.login(config['wiki_user'], config['wiki_password'])
125 generated_redirects = {}
127 for appid in sortedids:
128 app = metadata.App(apps[appid])
132 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
134 for af in app.AntiFeatures:
135 wikidata += '{{AntiFeature|' + af + '}}\n'
140 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
143 app.added.strftime('%Y-%m-%d') if app.added else '',
144 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
159 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
161 wikidata += app.Summary
162 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
164 wikidata += "=Description=\n"
165 wikidata += metadata.description_wiki(app.Description) + "\n"
167 wikidata += "=Maintainer Notes=\n"
168 if app.MaintainerNotes:
169 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
172 # Get a list of all packages for this application...
174 gotcurrentver = False
178 if apk['packageName'] == appid:
179 if str(apk['versionCode']) == app.CurrentVersionCode:
182 # Include ones we can't build, as a special case...
183 for build in app.builds:
185 if build.versionCode == app.CurrentVersionCode:
187 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188 apklist.append({'versionCode': int(build.versionCode),
189 'versionName': build.versionName,
190 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
195 if apk['versionCode'] == int(build.versionCode):
200 apklist.append({'versionCode': int(build.versionCode),
201 'versionName': build.versionName,
202 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
204 if app.CurrentVersionCode == '0':
206 # Sort with most recent first...
207 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
209 wikidata += "=Versions=\n"
210 if len(apklist) == 0:
211 wikidata += "We currently have no versions of this app available."
212 elif not gotcurrentver:
213 wikidata += "We don't have the current version of this app."
215 wikidata += "We have the current version of this app."
216 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218 if len(app.NoSourceSince) > 0:
219 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220 if len(app.CurrentVersion) > 0:
221 wikidata += "The current (recommended) version is " + app.CurrentVersion
222 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
225 wikidata += "==" + apk['versionName'] + "==\n"
227 if 'buildproblem' in apk:
228 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
231 wikidata += "This version is built and signed by "
233 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
235 wikidata += "the original developer.\n\n"
236 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
238 wikidata += '\n[[Category:' + wikicat + ']]\n'
239 if len(app.NoSourceSince) > 0:
240 wikidata += '\n[[Category:Apps missing source code]]\n'
241 if validapks == 0 and not app.Disabled:
242 wikidata += '\n[[Category:Apps with no packages]]\n'
243 if cantupdate and not app.Disabled:
244 wikidata += "\n[[Category:Apps we cannot update]]\n"
245 if buildfails and not app.Disabled:
246 wikidata += "\n[[Category:Apps with failing builds]]\n"
247 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248 wikidata += '\n[[Category:Apps to Update]]\n'
250 wikidata += '\n[[Category:Apps that are disabled]]\n'
251 if app.UpdateCheckMode == 'None' and not app.Disabled:
252 wikidata += '\n[[Category:Apps with no update check]]\n'
253 for appcat in app.Categories:
254 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
256 # We can't have underscores in the page name, even if they're in
257 # the package ID, because MediaWiki messes with them...
258 pagename = appid.replace('_', ' ')
260 # Drop a trailing newline, because mediawiki is going to drop it anyway
261 # and it we don't we'll think the page has changed when it hasn't...
262 if wikidata.endswith('\n'):
263 wikidata = wikidata[:-1]
265 generated_pages[pagename] = wikidata
267 # Make a redirect from the name to the ID too, unless there's
268 # already an existing page with the name and it isn't a redirect.
270 apppagename = app.Name.replace('_', ' ')
271 apppagename = apppagename.replace('{', '')
272 apppagename = apppagename.replace('}', ' ')
273 apppagename = apppagename.replace(':', ' ')
274 apppagename = apppagename.replace('[', ' ')
275 apppagename = apppagename.replace(']', ' ')
276 # Drop double spaces caused mostly by replacing ':' above
277 apppagename = apppagename.replace(' ', ' ')
278 for expagename in site.allpages(prefix=apppagename,
279 filterredir='nonredirects',
281 if expagename == apppagename:
283 # Another reason not to make the redirect page is if the app name
284 # is the same as it's ID, because that will overwrite the real page
285 # with an redirect to itself! (Although it seems like an odd
286 # scenario this happens a lot, e.g. where there is metadata but no
287 # builds or binaries to extract a name from.
288 if apppagename == pagename:
291 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
293 for tcat, genp in [(wikicat, generated_pages),
294 (wikiredircat, generated_redirects)]:
295 catpages = site.Pages['Category:' + tcat]
297 for page in catpages:
298 existingpages.append(page.name)
299 if page.name in genp:
300 pagetxt = page.edit()
301 if pagetxt != genp[page.name]:
302 logging.debug("Updating modified page " + page.name)
303 page.save(genp[page.name], summary='Auto-updated')
305 logging.debug("Page " + page.name + " is unchanged")
307 logging.warn("Deleting page " + page.name)
308 page.delete('No longer published')
309 for pagename, text in genp.items():
310 logging.debug("Checking " + pagename)
311 if pagename not in existingpages:
312 logging.debug("Creating page " + pagename)
314 newpage = site.Pages[pagename]
315 newpage.save(text, summary='Auto-created')
316 except Exception as e:
317 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
319 # Purge server cache to ensure counts are up to date
320 site.pages['Repository Maintenance'].purge()
323 def delete_disabled_builds(apps, apkcache, repodirs):
324 """Delete disabled build outputs.
326 :param apps: list of all applications, as per metadata.read_metadata
327 :param apkcache: current apk cache information
328 :param repodirs: the repo directories to process
330 for appid, app in apps.items():
331 for build in app['builds']:
332 if not build.disable:
334 apkfilename = common.get_release_filename(app, build)
335 iconfilename = "%s.%s.png" % (
338 for repodir in repodirs:
340 os.path.join(repodir, apkfilename),
341 os.path.join(repodir, apkfilename + '.asc'),
342 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
344 for density in all_screen_densities:
345 repo_dir = get_icon_dir(repodir, density)
346 files.append(os.path.join(repo_dir, iconfilename))
349 if os.path.exists(f):
350 logging.info("Deleting disabled build output " + f)
352 if apkfilename in apkcache:
353 del apkcache[apkfilename]
356 def resize_icon(iconpath, density):
358 if not os.path.isfile(iconpath):
363 fp = open(iconpath, 'rb')
365 size = dpi_to_px(density)
367 if any(length > size for length in im.size):
369 im.thumbnail((size, size), Image.ANTIALIAS)
370 logging.debug("%s was too large at %s - new size is %s" % (
371 iconpath, oldsize, im.size))
372 im.save(iconpath, "PNG")
374 except Exception as e:
375 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
382 def resize_all_icons(repodirs):
383 """Resize all icons that exceed the max size
385 :param repodirs: the repo directories to process
387 for repodir in repodirs:
388 for density in screen_densities:
389 icon_dir = get_icon_dir(repodir, density)
390 icon_glob = os.path.join(icon_dir, '*.png')
391 for iconpath in glob.glob(icon_glob):
392 resize_icon(iconpath, density)
396 """ Get the signing certificate of an apk. To get the same md5 has that
397 Android gets, we encode the .RSA certificate in a specific format and pass
398 it hex-encoded to the md5 digest algorithm.
400 :param apkpath: path to the apk
401 :returns: A string containing the md5 of the signature of the apk or None
402 if an error occurred.
405 with zipfile.ZipFile(apkpath, 'r') as apk:
406 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
409 logging.error("Found no signing certificates on %s" % apkpath)
412 logging.error("Found multiple signing certificates on %s" % apkpath)
415 cert = apk.read(certs[0])
417 cert_encoded = common.get_certificate(cert)
419 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
422 def get_cache_file():
423 return os.path.join('tmp', 'apkcache')
428 Gather information about all the apk files in the repo directory,
429 using cached data if possible.
432 apkcachefile = get_cache_file()
433 if not options.clean and os.path.exists(apkcachefile):
434 with open(apkcachefile, 'rb') as cf:
435 apkcache = pickle.load(cf, encoding='utf-8')
436 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
444 def write_cache(apkcache):
445 apkcachefile = get_cache_file()
446 cache_path = os.path.dirname(apkcachefile)
447 if not os.path.exists(cache_path):
448 os.makedirs(cache_path)
449 apkcache["METADATA_VERSION"] = METADATA_VERSION
450 with open(apkcachefile, 'wb') as cf:
451 pickle.dump(apkcache, cf)
454 def get_icon_bytes(apkzip, iconsrc):
455 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
457 return apkzip.read(iconsrc)
459 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
462 def sha256sum(filename):
463 '''Calculate the sha256 of the given file'''
464 sha = hashlib.sha256()
465 with open(filename, 'rb') as f:
471 return sha.hexdigest()
474 def has_old_openssl(filename):
475 '''checks for known vulnerable openssl versions in the APK'''
477 # statically load this pattern
478 if not hasattr(has_old_openssl, "pattern"):
479 has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
481 with zipfile.ZipFile(filename) as zf:
482 for name in zf.namelist():
483 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
486 chunk = lib.read(4096)
489 m = has_old_openssl.pattern.search(chunk)
491 version = m.group(1).decode('ascii')
492 if version.startswith('1.0.1') and version[5] >= 'r' \
493 or version.startswith('1.0.2') and version[5] >= 'f':
494 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
496 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
502 def insert_obbs(repodir, apps, apks):
503 """Scans the .obb files in a given repo directory and adds them to the
504 relevant APK instances. OBB files have versionCodes like APK
505 files, and they are loosely associated. If there is an OBB file
506 present, then any APK with the same or higher versionCode will use
507 that OBB file. There are two OBB types: main and patch, each APK
508 can only have only have one of each.
510 https://developer.android.com/google/play/expansion-files.html
512 :param repodir: repo directory to scan
513 :param apps: list of current, valid apps
514 :param apks: current information on all APKs
518 def obbWarnDelete(f, msg):
519 logging.warning(msg + f)
520 if options.delete_unknown:
521 logging.error("Deleting unknown file: " + f)
525 java_Integer_MIN_VALUE = -pow(2, 31)
526 currentPackageNames = apps.keys()
527 for f in glob.glob(os.path.join(repodir, '*.obb')):
528 obbfile = os.path.basename(f)
529 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
530 chunks = obbfile.split('.')
531 if chunks[0] != 'main' and chunks[0] != 'patch':
532 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
534 if not re.match(r'^-?[0-9]+$', chunks[1]):
535 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
537 versionCode = int(chunks[1])
538 packagename = ".".join(chunks[2:-1])
540 highestVersionCode = java_Integer_MIN_VALUE
541 if packagename not in currentPackageNames:
542 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
545 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
546 highestVersionCode = apk['versionCode']
547 if versionCode > highestVersionCode:
548 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
549 + ') than any APK: ')
551 obbsha256 = sha256sum(f)
552 obbs.append((packagename, versionCode, obbfile, obbsha256))
555 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
556 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
557 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
558 apk['obbMainFile'] = obbfile
559 apk['obbMainFileSha256'] = obbsha256
560 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
561 apk['obbPatchFile'] = obbfile
562 apk['obbPatchFileSha256'] = obbsha256
563 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
567 def _get_localized_dict(app, locale):
568 '''get the dict to add localized store metadata to'''
569 if 'localized' not in app:
570 app['localized'] = collections.OrderedDict()
571 if locale not in app['localized']:
572 app['localized'][locale] = collections.OrderedDict()
573 return app['localized'][locale]
576 def _set_localized_text_entry(app, locale, key, f):
577 limit = config['char_limits'][key]
578 localized = _get_localized_dict(app, locale)
580 text = fp.read()[:limit]
582 localized[key] = text
585 def _set_author_entry(app, key, f):
586 limit = config['char_limits']['author']
588 text = fp.read()[:limit]
593 def copy_triple_t_store_metadata(apps):
594 """Include store metadata from the app's source repo
596 The Triple-T Gradle Play Publisher is a plugin that has a standard
597 file layout for all of the metadata and graphics that the Google
598 Play Store accepts. Since F-Droid has the git repo, it can just
599 pluck those files directly. This method reads any text files into
600 the app dict, then copies any graphics into the fdroid repo
603 This needs to be run before insert_localized_app_metadata() so that
604 the graphics files that are copied into the fdroid repo get
607 https://github.com/Triple-T/gradle-play-publisher#upload-images
608 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
612 if not os.path.isdir('build'):
613 return # nothing to do
615 for packageName, app in apps.items():
616 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
617 logging.debug('Triple-T Gradle Play Publisher: ' + d)
618 for root, dirs, files in os.walk(d):
619 segments = root.split('/')
620 locale = segments[-2]
622 if f == 'fulldescription':
623 _set_localized_text_entry(app, locale, 'description',
624 os.path.join(root, f))
626 elif f == 'shortdescription':
627 _set_localized_text_entry(app, locale, 'summary',
628 os.path.join(root, f))
631 _set_localized_text_entry(app, locale, 'name',
632 os.path.join(root, f))
635 _set_localized_text_entry(app, locale, 'video',
636 os.path.join(root, f))
638 elif f == 'whatsnew':
639 _set_localized_text_entry(app, segments[-1], 'whatsNew',
640 os.path.join(root, f))
642 elif f == 'contactEmail':
643 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
645 elif f == 'contactPhone':
646 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
648 elif f == 'contactWebsite':
649 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
652 base, extension = common.get_extension(f)
653 dirname = os.path.basename(root)
654 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
655 if segments[-2] == 'listing':
656 locale = segments[-3]
658 locale = segments[-2]
659 destdir = os.path.join('repo', packageName, locale)
660 os.makedirs(destdir, mode=0o755, exist_ok=True)
661 sourcefile = os.path.join(root, f)
662 destfile = os.path.join(destdir, dirname + '.' + extension)
663 logging.debug('copying ' + sourcefile + ' ' + destfile)
664 shutil.copy(sourcefile, destfile)
667 def insert_localized_app_metadata(apps):
668 """scans standard locations for graphics and localized text
670 Scans for localized description files, store graphics, and
671 screenshot PNG files in statically defined screenshots directory
672 and adds them to the app metadata. The screenshots and graphic
673 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
674 and must be in the following layout:
675 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
677 repo/packageName/locale/featureGraphic.png
678 repo/packageName/locale/phoneScreenshots/1.png
679 repo/packageName/locale/phoneScreenshots/2.png
681 The changelog files must be text files named with the versionCode
682 ending with ".txt" and must be in the following layout:
683 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
685 repo/packageName/locale/changelogs/12345.txt
687 This will scan the each app's source repo then the metadata/ dir
688 for these standard locations of changelog files. If it finds
689 them, they will be added to the dict of all packages, with the
690 versions in the metadata/ folder taking precendence over the what
691 is in the app's source repo.
693 Where "packageName" is the app's packageName and "locale" is the locale
694 of the graphics, e.g. what language they are in, using the IETF RFC5646
695 format (en-US, fr-CA, es-MX, etc).
697 This will also scan the app's git for a fastlane folder, and the
698 metadata/ folder and the apps' source repos for standard locations
699 of graphic and screenshot files. If it finds them, it will copy
700 them into the repo. The fastlane files follow this pattern:
701 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
705 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
706 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
708 for d in sorted(sourcedirs):
709 if not os.path.isdir(d):
711 for root, dirs, files in os.walk(d):
712 segments = root.split('/')
713 packageName = segments[1]
714 if packageName not in apps:
715 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
717 locale = segments[-1]
719 if f == 'full_description.txt':
720 _set_localized_text_entry(apps[packageName], locale, 'description',
721 os.path.join(root, f))
723 elif f == 'short_description.txt':
724 _set_localized_text_entry(apps[packageName], locale, 'summary',
725 os.path.join(root, f))
727 elif f == 'title.txt':
728 _set_localized_text_entry(apps[packageName], locale, 'name',
729 os.path.join(root, f))
731 elif f == 'video.txt':
732 _set_localized_text_entry(apps[packageName], locale, 'video',
733 os.path.join(root, f))
735 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
736 locale = segments[-2]
737 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
738 os.path.join(root, f))
741 base, extension = common.get_extension(f)
742 if locale == 'images':
743 locale = segments[-2]
744 destdir = os.path.join('repo', packageName, locale)
745 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
746 os.makedirs(destdir, mode=0o755, exist_ok=True)
747 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
748 shutil.copy(os.path.join(root, f), destdir)
750 if d in SCREENSHOT_DIRS:
751 for f in glob.glob(os.path.join(root, d, '*.*')):
752 _, extension = common.get_extension(f)
753 if extension in ALLOWED_EXTENSIONS:
754 screenshotdestdir = os.path.join(destdir, d)
755 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
756 logging.debug('copying ' + f + ' ' + screenshotdestdir)
757 shutil.copy(f, screenshotdestdir)
759 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
761 if not os.path.isdir(d):
763 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
764 if not os.path.isfile(f):
766 segments = f.split('/')
767 packageName = segments[1]
769 screenshotdir = segments[3]
770 filename = os.path.basename(f)
771 base, extension = common.get_extension(filename)
773 if packageName not in apps:
774 logging.warning('Found "%s" graphic without metadata for app "%s"!'
775 % (filename, packageName))
777 graphics = _get_localized_dict(apps[packageName], locale)
779 if extension not in ALLOWED_EXTENSIONS:
780 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
781 elif base in GRAPHIC_NAMES:
782 # there can only be zero or one of these per locale
783 graphics[base] = filename
784 elif screenshotdir in SCREENSHOT_DIRS:
785 # there can any number of these per locale
786 logging.debug('adding to ' + screenshotdir + ': ' + f)
787 if screenshotdir not in graphics:
788 graphics[screenshotdir] = []
789 graphics[screenshotdir].append(filename)
791 logging.warning('Unsupported graphics file found: ' + f)
794 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
795 """Scan a repo for all files with an extension except APK/OBB
797 :param apkcache: current cached info about all repo files
798 :param repodir: repo directory to scan
799 :param knownapks: list of all known files, as per metadata.read_metadata
800 :param use_date_from_file: use date from file (instead of current date)
801 for newly added files
806 repodir = repodir.encode('utf-8')
807 for name in os.listdir(repodir):
808 file_extension = common.get_file_extension(name)
809 if file_extension == 'apk' or file_extension == 'obb':
811 filename = os.path.join(repodir, name)
812 name_utf8 = name.decode('utf-8')
813 if filename.endswith(b'_src.tar.gz'):
814 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
816 if not common.is_repo_file(filename):
818 stat = os.stat(filename)
819 if stat.st_size == 0:
820 raise FDroidException(filename + ' is zero size!')
822 shasum = sha256sum(filename)
825 repo_file = apkcache[name]
826 # added time is cached as tuple but used here as datetime instance
827 if 'added' in repo_file:
828 a = repo_file['added']
829 if isinstance(a, datetime):
830 repo_file['added'] = a
832 repo_file['added'] = datetime(*a[:6])
833 if repo_file.get('hash') == shasum:
834 logging.debug("Reading " + name_utf8 + " from cache")
837 logging.debug("Ignoring stale cache data for " + name)
840 logging.debug("Processing " + name_utf8)
841 repo_file = collections.OrderedDict()
842 repo_file['name'] = os.path.splitext(name_utf8)[0]
843 # TODO rename apkname globally to something more generic
844 repo_file['apkName'] = name_utf8
845 repo_file['hash'] = shasum
846 repo_file['hashType'] = 'sha256'
847 repo_file['versionCode'] = 0
848 repo_file['versionName'] = shasum
849 # the static ID is the SHA256 unless it is set in the metadata
850 repo_file['packageName'] = shasum
852 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
854 repo_file['packageName'] = m.group(1)
855 repo_file['versionCode'] = int(m.group(2))
856 srcfilename = name + b'_src.tar.gz'
857 if os.path.exists(os.path.join(repodir, srcfilename)):
858 repo_file['srcname'] = srcfilename.decode('utf-8')
859 repo_file['size'] = stat.st_size
861 apkcache[name] = repo_file
864 if use_date_from_file:
865 timestamp = stat.st_ctime
866 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
868 default_date_param = None
870 # Record in knownapks, getting the added date at the same time..
871 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
872 default_date=default_date_param)
874 repo_file['added'] = added
876 repo_files.append(repo_file)
878 return repo_files, cachechanged
881 def scan_apk_aapt(apk, apkfile):
882 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
883 if p.returncode != 0:
884 if options.delete_unknown:
885 if os.path.exists(apkfile):
886 logging.error("Failed to get apk information, deleting " + apkfile)
889 logging.error("Could not find {0} to remove it".format(apkfile))
891 logging.error("Failed to get apk information, skipping " + apkfile)
892 raise BuildException("Invalid APK")
893 for line in p.output.splitlines():
894 if line.startswith("package:"):
896 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
897 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
898 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
899 except Exception as e:
900 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
901 elif line.startswith("application:"):
902 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
903 # Keep path to non-dpi icon in case we need it
904 match = re.match(APK_ICON_PAT_NODPI, line)
906 apk['icons_src']['-1'] = match.group(1)
907 elif line.startswith("launchable-activity:"):
908 # Only use launchable-activity as fallback to application
910 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
911 if '-1' not in apk['icons_src']:
912 match = re.match(APK_ICON_PAT_NODPI, line)
914 apk['icons_src']['-1'] = match.group(1)
915 elif line.startswith("application-icon-"):
916 match = re.match(APK_ICON_PAT, line)
918 density = match.group(1)
919 path = match.group(2)
920 apk['icons_src'][density] = path
921 elif line.startswith("sdkVersion:"):
922 m = re.match(APK_SDK_VERSION_PAT, line)
924 logging.error(line.replace('sdkVersion:', '')
925 + ' is not a valid minSdkVersion!')
927 apk['minSdkVersion'] = m.group(1)
928 # if target not set, default to min
929 if 'targetSdkVersion' not in apk:
930 apk['targetSdkVersion'] = m.group(1)
931 elif line.startswith("targetSdkVersion:"):
932 m = re.match(APK_SDK_VERSION_PAT, line)
934 logging.error(line.replace('targetSdkVersion:', '')
935 + ' is not a valid targetSdkVersion!')
937 apk['targetSdkVersion'] = m.group(1)
938 elif line.startswith("maxSdkVersion:"):
939 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
940 elif line.startswith("native-code:"):
941 apk['nativecode'] = []
942 for arch in line[13:].split(' '):
943 apk['nativecode'].append(arch[1:-1])
944 elif line.startswith('uses-permission:'):
945 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
946 if perm_match['maxSdkVersion']:
947 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
948 permission = UsesPermission(
950 perm_match['maxSdkVersion']
953 apk['uses-permission'].append(permission)
954 elif line.startswith('uses-permission-sdk-23:'):
955 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
956 if perm_match['maxSdkVersion']:
957 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
958 permission_sdk_23 = UsesPermissionSdk23(
960 perm_match['maxSdkVersion']
963 apk['uses-permission-sdk-23'].append(permission_sdk_23)
965 elif line.startswith('uses-feature:'):
966 feature = re.match(APK_FEATURE_PAT, line).group(1)
967 # Filter out this, it's only added with the latest SDK tools and
968 # causes problems for lots of apps.
969 if feature != "android.hardware.screen.portrait" \
970 and feature != "android.hardware.screen.landscape":
971 if feature.startswith("android.feature."):
972 feature = feature[16:]
973 apk['features'].add(feature)
976 def scan_apk_androguard(apk, apkfile):
978 from androguard.core.bytecodes.apk import APK
979 apkobject = APK(apkfile)
980 if apkobject.is_valid_APK():
981 arsc = apkobject.get_android_resources()
983 if options.delete_unknown:
984 if os.path.exists(apkfile):
985 logging.error("Failed to get apk information, deleting " + apkfile)
988 logging.error("Could not find {0} to remove it".format(apkfile))
990 logging.error("Failed to get apk information, skipping " + apkfile)
991 raise BuildException("Invaild APK")
993 raise FDroidException("androguard library is not installed and aapt not present")
994 except FileNotFoundError:
995 logging.error("Could not open apk file for analysis")
996 raise BuildException("Invalid APK")
998 apk['packageName'] = apkobject.get_package()
999 apk['versionCode'] = int(apkobject.get_androidversion_code())
1000 apk['versionName'] = apkobject.get_androidversion_name()
1001 if apk['versionName'][0] == "@":
1002 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1003 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1004 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1005 apk['name'] = apkobject.get_app_name()
1007 if apkobject.get_max_sdk_version() is not None:
1008 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1009 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1010 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1012 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1013 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1015 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1017 for file in apkobject.get_files():
1018 d_re = density_re.match(file)
1020 folder = d_re.group(1).split('-')
1022 resolution = folder[1]
1025 density = screen_resolutions[resolution]
1026 apk['icons_src'][density] = d_re.group(0)
1028 if apk['icons_src'].get('-1') is None:
1029 apk['icons_src']['-1'] = apk['icons_src']['160']
1031 arch_re = re.compile("^lib/(.*)/.*$")
1032 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1034 apk['nativecode'] = []
1035 apk['nativecode'].extend(sorted(list(arch)))
1037 xml = apkobject.get_android_manifest_xml()
1039 for item in xml.getElementsByTagName('uses-permission'):
1040 name = str(item.getAttribute("android:name"))
1041 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1042 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1043 permission = UsesPermission(
1047 apk['uses-permission'].append(permission)
1049 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1050 name = str(item.getAttribute("android:name"))
1051 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1052 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1053 permission_sdk_23 = UsesPermissionSdk23(
1057 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1059 for item in xml.getElementsByTagName('uses-feature'):
1060 feature = str(item.getAttribute("android:name"))
1061 if feature != "android.hardware.screen.portrait" \
1062 and feature != "android.hardware.screen.landscape":
1063 if feature.startswith("android.feature."):
1064 feature = feature[16:]
1065 apk['features'].append(feature)
1068 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1069 """Scan the apk with the given filename in the given repo directory.
1071 This also extracts the icons.
1073 :param apkcache: current apk cache information
1074 :param apkfilename: the filename of the apk to scan
1075 :param repodir: repo directory to scan
1076 :param knownapks: known apks info
1077 :param use_date_from_apk: use date from APK (instead of current date)
1078 for newly added APKs
1079 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1080 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1083 if ' ' in apkfilename:
1084 if options.rename_apks:
1085 newfilename = apkfilename.replace(' ', '_')
1086 os.rename(os.path.join(repodir, apkfilename),
1087 os.path.join(repodir, newfilename))
1088 apkfilename = newfilename
1090 logging.critical("Spaces in filenames are not allowed.")
1091 return True, None, False
1093 apkfile = os.path.join(repodir, apkfilename)
1094 shasum = sha256sum(apkfile)
1096 cachechanged = False
1098 if apkfilename in apkcache:
1099 apk = apkcache[apkfilename]
1100 if apk.get('hash') == shasum:
1101 logging.debug("Reading " + apkfilename + " from cache")
1104 logging.debug("Ignoring stale cache data for " + apkfilename)
1107 logging.debug("Processing " + apkfilename)
1109 apk['hash'] = shasum
1110 apk['hashType'] = 'sha256'
1111 apk['uses-permission'] = []
1112 apk['uses-permission-sdk-23'] = []
1113 apk['features'] = []
1114 apk['icons_src'] = {}
1116 apk['antiFeatures'] = set()
1119 if SdkToolsPopen(['aapt', 'version'], output=False):
1120 scan_apk_aapt(apk, apkfile)
1122 scan_apk_androguard(apk, apkfile)
1123 except BuildException:
1124 return True, None, False
1126 if 'minSdkVersion' not in apk:
1127 logging.warn("No SDK version information found in {0}".format(apkfile))
1128 apk['minSdkVersion'] = 1
1130 # Check for debuggable apks...
1131 if common.isApkAndDebuggable(apkfile):
1132 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1134 # Get the signature (or md5 of, to be precise)...
1135 logging.debug('Getting signature of {0}'.format(apkfile))
1136 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1138 logging.critical("Failed to get apk signature")
1139 return True, None, False
1141 if options.rename_apks:
1142 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1143 std_short_name = os.path.join(repodir, n)
1144 if apkfile != std_short_name:
1145 if os.path.exists(std_short_name):
1146 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1147 if apkfile != std_long_name:
1148 if os.path.exists(std_long_name):
1149 dupdir = os.path.join('duplicates', repodir)
1150 if not os.path.isdir(dupdir):
1151 os.makedirs(dupdir, exist_ok=True)
1152 dupfile = os.path.join('duplicates', std_long_name)
1153 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1154 os.rename(apkfile, dupfile)
1155 return True, None, False
1157 os.rename(apkfile, std_long_name)
1158 apkfile = std_long_name
1160 os.rename(apkfile, std_short_name)
1161 apkfile = std_short_name
1162 apkfilename = apkfile[len(repodir) + 1:]
1164 apk['apkName'] = apkfilename
1165 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1166 if os.path.exists(os.path.join(repodir, srcfilename)):
1167 apk['srcname'] = srcfilename
1168 apk['size'] = os.path.getsize(apkfile)
1170 # verify the jar signature is correct
1171 if not common.verify_apk_signature(apkfile):
1172 return True, None, False
1174 if has_old_openssl(apkfile):
1175 apk['antiFeatures'].add('KnownVuln')
1177 apkzip = zipfile.ZipFile(apkfile, 'r')
1179 # if an APK has files newer than the system time, suggest updating
1180 # the system clock. This is useful for offline systems, used for
1181 # signing, which do not have another source of clock sync info. It
1182 # has to be more than 24 hours newer because ZIP/APK files do not
1183 # store timezone info
1184 manifest = apkzip.getinfo('AndroidManifest.xml')
1185 if manifest.date_time[1] == 0: # month can't be zero
1186 logging.debug('AndroidManifest.xml has no date')
1188 dt_obj = datetime(*manifest.date_time)
1189 checkdt = dt_obj - timedelta(1)
1190 if datetime.today() < checkdt:
1191 logging.warn('System clock is older than manifest in: '
1193 + '\nSet clock to that time using:\n'
1194 + 'sudo date -s "' + str(dt_obj) + '"')
1196 iconfilename = "%s.%s.png" % (
1200 # Extract the icon file...
1201 empty_densities = []
1202 for density in screen_densities:
1203 if density not in apk['icons_src']:
1204 empty_densities.append(density)
1206 iconsrc = apk['icons_src'][density]
1207 icon_dir = get_icon_dir(repodir, density)
1208 icondest = os.path.join(icon_dir, iconfilename)
1211 with open(icondest, 'wb') as f:
1212 f.write(get_icon_bytes(apkzip, iconsrc))
1213 apk['icons'][density] = iconfilename
1214 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1215 logging.warning("Error retrieving icon file: %s" % (icondest))
1216 del apk['icons_src'][density]
1217 empty_densities.append(density)
1219 if '-1' in apk['icons_src']:
1220 iconsrc = apk['icons_src']['-1']
1221 iconpath = os.path.join(
1222 get_icon_dir(repodir, '0'), iconfilename)
1223 with open(iconpath, 'wb') as f:
1224 f.write(get_icon_bytes(apkzip, iconsrc))
1226 im = Image.open(iconpath)
1227 dpi = px_to_dpi(im.size[0])
1228 for density in screen_densities:
1229 if density in apk['icons']:
1231 if density == screen_densities[-1] or dpi >= int(density):
1232 apk['icons'][density] = iconfilename
1233 shutil.move(iconpath,
1234 os.path.join(get_icon_dir(repodir, density), iconfilename))
1235 empty_densities.remove(density)
1237 except Exception as e:
1238 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1241 apk['icon'] = iconfilename
1245 # First try resizing down to not lose quality
1247 for density in screen_densities:
1248 if density not in empty_densities:
1249 last_density = density
1251 if last_density is None:
1253 logging.debug("Density %s not available, resizing down from %s"
1254 % (density, last_density))
1256 last_iconpath = os.path.join(
1257 get_icon_dir(repodir, last_density), iconfilename)
1258 iconpath = os.path.join(
1259 get_icon_dir(repodir, density), iconfilename)
1262 fp = open(last_iconpath, 'rb')
1265 size = dpi_to_px(density)
1267 im.thumbnail((size, size), Image.ANTIALIAS)
1268 im.save(iconpath, "PNG")
1269 empty_densities.remove(density)
1270 except Exception as e:
1271 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1276 # Then just copy from the highest resolution available
1278 for density in reversed(screen_densities):
1279 if density not in empty_densities:
1280 last_density = density
1282 if last_density is None:
1284 logging.debug("Density %s not available, copying from lower density %s"
1285 % (density, last_density))
1288 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1289 os.path.join(get_icon_dir(repodir, density), iconfilename))
1291 empty_densities.remove(density)
1293 for density in screen_densities:
1294 icon_dir = get_icon_dir(repodir, density)
1295 icondest = os.path.join(icon_dir, iconfilename)
1296 resize_icon(icondest, density)
1298 # Copy from icons-mdpi to icons since mdpi is the baseline density
1299 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1300 if os.path.isfile(baseline):
1301 apk['icons']['0'] = iconfilename
1302 shutil.copyfile(baseline,
1303 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1305 if use_date_from_apk and manifest.date_time[1] != 0:
1306 default_date_param = datetime(*manifest.date_time)
1308 default_date_param = None
1310 # Record in known apks, getting the added date at the same time..
1311 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1312 default_date=default_date_param)
1314 apk['added'] = added
1316 apkcache[apkfilename] = apk
1319 return False, apk, cachechanged
1322 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1323 """Scan the apks in the given repo directory.
1325 This also extracts the icons.
1327 :param apkcache: current apk cache information
1328 :param repodir: repo directory to scan
1329 :param knownapks: known apks info
1330 :param use_date_from_apk: use date from APK (instead of current date)
1331 for newly added APKs
1332 :returns: (apks, cachechanged) where apks is a list of apk information,
1333 and cachechanged is True if the apkcache got changed.
1336 cachechanged = False
1338 for icon_dir in get_all_icon_dirs(repodir):
1339 if os.path.exists(icon_dir):
1341 shutil.rmtree(icon_dir)
1342 os.makedirs(icon_dir)
1344 os.makedirs(icon_dir)
1347 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1348 apkfilename = apkfile[len(repodir) + 1:]
1349 (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1354 return apks, cachechanged
1357 def apply_info_from_latest_apk(apps, apks):
1359 Some information from the apks needs to be applied up to the application level.
1360 When doing this, we use the info from the most recent version's apk.
1361 We deal with figuring out when the app was added and last updated at the same time.
1363 for appid, app in apps.items():
1364 bestver = UNSET_VERSION_CODE
1366 if apk['packageName'] == appid:
1367 if apk['versionCode'] > bestver:
1368 bestver = apk['versionCode']
1372 if not app.added or apk['added'] < app.added:
1373 app.added = apk['added']
1374 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1375 app.lastUpdated = apk['added']
1378 logging.debug("Don't know when " + appid + " was added")
1379 if not app.lastUpdated:
1380 logging.debug("Don't know when " + appid + " was last updated")
1382 if bestver == UNSET_VERSION_CODE:
1384 if app.Name is None:
1385 app.Name = app.AutoName or appid
1387 logging.debug("Application " + appid + " has no packages")
1389 if app.Name is None:
1390 app.Name = bestapk['name']
1391 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1392 if app.CurrentVersionCode is None:
1393 app.CurrentVersionCode = str(bestver)
1396 def make_categories_txt(repodir, categories):
1397 '''Write a category list in the repo to allow quick access'''
1399 for cat in sorted(categories):
1400 catdata += cat + '\n'
1401 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1405 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1407 for appid, app in apps.items():
1409 if app.ArchivePolicy:
1410 keepversions = int(app.ArchivePolicy[:-9])
1412 keepversions = defaultkeepversions
1414 def filter_apk_list_sorted(apk_list):
1416 for apk in apk_list:
1417 if apk['packageName'] == appid:
1420 # Sort the apk list by version code. First is highest/newest.
1421 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1423 def move_file(from_dir, to_dir, filename, ignore_missing):
1424 from_path = os.path.join(from_dir, filename)
1425 if ignore_missing and not os.path.exists(from_path):
1427 to_path = os.path.join(to_dir, filename)
1428 shutil.move(from_path, to_path)
1430 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1431 .format(appid, len(apks), keepversions, len(archapks)))
1433 if len(apks) > keepversions:
1434 apklist = filter_apk_list_sorted(apks)
1435 # Move back the ones we don't want.
1436 for apk in apklist[keepversions:]:
1437 logging.info("Moving " + apk['apkName'] + " to archive")
1438 move_file(repodir, archivedir, apk['apkName'], False)
1439 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1440 for density in all_screen_densities:
1441 repo_icon_dir = get_icon_dir(repodir, density)
1442 archive_icon_dir = get_icon_dir(archivedir, density)
1443 if density not in apk['icons']:
1445 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1446 if 'srcname' in apk:
1447 move_file(repodir, archivedir, apk['srcname'], False)
1448 archapks.append(apk)
1450 elif len(apks) < keepversions and len(archapks) > 0:
1451 required = keepversions - len(apks)
1452 archapklist = filter_apk_list_sorted(archapks)
1453 # Move forward the ones we want again.
1454 for apk in archapklist[:required]:
1455 logging.info("Moving " + apk['apkName'] + " from archive")
1456 move_file(archivedir, repodir, apk['apkName'], False)
1457 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1458 for density in all_screen_densities:
1459 repo_icon_dir = get_icon_dir(repodir, density)
1460 archive_icon_dir = get_icon_dir(archivedir, density)
1461 if density not in apk['icons']:
1463 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1464 if 'srcname' in apk:
1465 move_file(archivedir, repodir, apk['srcname'], False)
1466 archapks.remove(apk)
1470 def add_apks_to_per_app_repos(repodir, apks):
1471 apks_per_app = dict()
1473 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1474 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1475 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1476 apks_per_app[apk['packageName']] = apk
1478 if not os.path.exists(apk['per_app_icons']):
1479 logging.info('Adding new repo for only ' + apk['packageName'])
1480 os.makedirs(apk['per_app_icons'])
1482 apkpath = os.path.join(repodir, apk['apkName'])
1483 shutil.copy(apkpath, apk['per_app_repo'])
1484 apksigpath = apkpath + '.sig'
1485 if os.path.exists(apksigpath):
1486 shutil.copy(apksigpath, apk['per_app_repo'])
1487 apkascpath = apkpath + '.asc'
1488 if os.path.exists(apkascpath):
1489 shutil.copy(apkascpath, apk['per_app_repo'])
1498 global config, options
1500 # Parse command line...
1501 parser = ArgumentParser()
1502 common.setup_global_opts(parser)
1503 parser.add_argument("--create-key", action="store_true", default=False,
1504 help="Create a repo signing key in a keystore")
1505 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1506 help="Create skeleton metadata files that are missing")
1507 parser.add_argument("--delete-unknown", action="store_true", default=False,
1508 help="Delete APKs and/or OBBs without metadata from the repo")
1509 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1510 help="Report on build data status")
1511 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1512 help="Interactively ask about things that need updating.")
1513 parser.add_argument("-I", "--icons", action="store_true", default=False,
1514 help="Resize all the icons exceeding the max pixel size and exit")
1515 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1516 help="Specify editor to use in interactive mode. Default " +
1517 "is /etc/alternatives/editor")
1518 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1519 help="Update the wiki")
1520 parser.add_argument("--pretty", action="store_true", default=False,
1521 help="Produce human-readable index.xml")
1522 parser.add_argument("--clean", action="store_true", default=False,
1523 help="Clean update - don't uses caches, reprocess all apks")
1524 parser.add_argument("--nosign", action="store_true", default=False,
1525 help="When configured for signed indexes, create only unsigned indexes at this stage")
1526 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1527 help="Use date from apk instead of current time for newly added apks")
1528 parser.add_argument("--rename-apks", action="store_true", default=False,
1529 help="Rename APK files that do not match package.name_123.apk")
1530 metadata.add_metadata_arguments(parser)
1531 options = parser.parse_args()
1532 metadata.warnings_action = options.W
1534 config = common.read_config(options)
1536 if not ('jarsigner' in config and 'keytool' in config):
1537 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1540 if config['archive_older'] != 0:
1541 repodirs.append('archive')
1542 if not os.path.exists('archive'):
1546 resize_all_icons(repodirs)
1549 if options.rename_apks:
1550 options.clean = True
1552 # check that icons exist now, rather than fail at the end of `fdroid update`
1553 for k in ['repo_icon', 'archive_icon']:
1555 if not os.path.exists(config[k]):
1556 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1559 # if the user asks to create a keystore, do it now, reusing whatever it can
1560 if options.create_key:
1561 if os.path.exists(config['keystore']):
1562 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1563 logging.critical("\t'" + config['keystore'] + "'")
1566 if 'repo_keyalias' not in config:
1567 config['repo_keyalias'] = socket.getfqdn()
1568 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1569 if 'keydname' not in config:
1570 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1571 common.write_to_config(config, 'keydname', config['keydname'])
1572 if 'keystore' not in config:
1573 config['keystore'] = common.default_config['keystore']
1574 common.write_to_config(config, 'keystore', config['keystore'])
1576 password = common.genpassword()
1577 if 'keystorepass' not in config:
1578 config['keystorepass'] = password
1579 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1580 if 'keypass' not in config:
1581 config['keypass'] = password
1582 common.write_to_config(config, 'keypass', config['keypass'])
1583 common.genkeystore(config)
1586 apps = metadata.read_metadata()
1588 # Generate a list of categories...
1590 for app in apps.values():
1591 categories.update(app.Categories)
1593 # Read known apks data (will be updated and written back when we've finished)
1594 knownapks = common.KnownApks()
1597 apkcache = get_cache()
1599 # Delete builds for disabled apps
1600 delete_disabled_builds(apps, apkcache, repodirs)
1602 # Scan all apks in the main repo
1603 apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1605 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1606 options.use_date_from_apk)
1607 cachechanged = cachechanged or fcachechanged
1609 # Generate warnings for apk's with no metadata (or create skeleton
1610 # metadata files, if requested on the command line)
1613 if apk['packageName'] not in apps:
1614 if options.create_metadata:
1615 if 'name' not in apk:
1616 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1618 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1619 f.write("License:Unknown\n")
1620 f.write("Web Site:\n")
1621 f.write("Source Code:\n")
1622 f.write("Issue Tracker:\n")
1623 f.write("Changelog:\n")
1624 f.write("Summary:" + apk['name'] + "\n")
1625 f.write("Description:\n")
1626 f.write(apk['name'] + "\n")
1628 f.write("Name:" + apk['name'] + "\n")
1630 logging.info("Generated skeleton metadata for " + apk['packageName'])
1633 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1634 if options.delete_unknown:
1635 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1636 rmf = os.path.join(repodirs[0], apk['apkName'])
1637 if not os.path.exists(rmf):
1638 logging.error("Could not find {0} to remove it".format(rmf))
1642 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1644 # update the metadata with the newly created ones included
1646 apps = metadata.read_metadata()
1648 copy_triple_t_store_metadata(apps)
1649 insert_obbs(repodirs[0], apps, apks)
1650 insert_localized_app_metadata(apps)
1652 # Scan the archive repo for apks as well
1653 if len(repodirs) > 1:
1654 archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1660 # Apply information from latest apks to the application and update dates
1661 apply_info_from_latest_apk(apps, apks + archapks)
1663 # Sort the app list by name, then the web site doesn't have to by default.
1664 # (we had to wait until we'd scanned the apks to do this, because mostly the
1665 # name comes from there!)
1666 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1668 # APKs are placed into multiple repos based on the app package, providing
1669 # per-app subscription feeds for nightly builds and things like it
1670 if config['per_app_repos']:
1671 add_apks_to_per_app_repos(repodirs[0], apks)
1672 for appid, app in apps.items():
1673 repodir = os.path.join(appid, 'fdroid', 'repo')
1675 appdict[appid] = app
1676 if os.path.isdir(repodir):
1677 index.make(appdict, [appid], apks, repodir, False)
1679 logging.info('Skipping index generation for ' + appid)
1682 if len(repodirs) > 1:
1683 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1685 # Make the index for the main repo...
1686 index.make(apps, sortedids, apks, repodirs[0], False)
1687 make_categories_txt(repodirs[0], categories)
1689 # If there's an archive repo, make the index for it. We already scanned it
1691 if len(repodirs) > 1:
1692 index.make(apps, sortedids, archapks, repodirs[1], True)
1694 git_remote = config.get('binary_transparency_remote')
1695 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1696 btlog.make_binary_transparency_log(repodirs)
1698 if config['update_stats']:
1699 # Update known apks info...
1700 knownapks.writeifchanged()
1702 # Generate latest apps data for widget
1703 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1705 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1707 appid = line.rstrip()
1708 data += appid + "\t"
1710 data += app.Name + "\t"
1711 if app.icon is not None:
1712 data += app.icon + "\t"
1713 data += app.License + "\n"
1714 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1718 write_cache(apkcache)
1720 # Update the wiki...
1722 update_wiki(apps, sortedids, apks + archapks)
1724 logging.info("Finished.")
1727 if __name__ == "__main__":