3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
35 from binascii import hexlify
42 from . import metadata
43 from .common import SdkToolsPopen
44 from .exception import BuildException, FDroidException
48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
49 UNSET_VERSION_CODE = -0x100000000
51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
58 APK_PERMISSION_PAT = \
59 re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
62 screen_densities = ['640', '480', '320', '240', '160', '120']
63 screen_resolutions = {
75 all_screen_densities = ['0'] + screen_densities
77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
83 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
86 def dpi_to_px(density):
87 return (int(density) * 48) / 160
91 return (int(px) * 160) / 48
94 def get_icon_dir(repodir, density):
96 return os.path.join(repodir, "icons")
97 return os.path.join(repodir, "icons-%s" % density)
100 def get_icon_dirs(repodir):
101 for density in screen_densities:
102 yield get_icon_dir(repodir, density)
105 def get_all_icon_dirs(repodir):
106 for density in all_screen_densities:
107 yield get_icon_dir(repodir, density)
110 def update_wiki(apps, sortedids, apks):
113 :param apps: fully populated list of all applications
114 :param apks: all apks, except...
116 logging.info("Updating wiki")
118 wikiredircat = 'App Redirects'
120 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
121 path=config['wiki_path'])
122 site.login(config['wiki_user'], config['wiki_password'])
124 generated_redirects = {}
126 for appid in sortedids:
127 app = metadata.App(apps[appid])
131 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
133 for af in app.AntiFeatures:
134 wikidata += '{{AntiFeature|' + af + '}}\n'
139 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
142 app.added.strftime('%Y-%m-%d') if app.added else '',
143 app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
158 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
160 wikidata += app.Summary
161 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
163 wikidata += "=Description=\n"
164 wikidata += metadata.description_wiki(app.Description) + "\n"
166 wikidata += "=Maintainer Notes=\n"
167 if app.MaintainerNotes:
168 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
169 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
171 # Get a list of all packages for this application...
173 gotcurrentver = False
177 if apk['packageName'] == appid:
178 if str(apk['versionCode']) == app.CurrentVersionCode:
181 # Include ones we can't build, as a special case...
182 for build in app.builds:
184 if build.versionCode == app.CurrentVersionCode:
186 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
187 apklist.append({'versionCode': int(build.versionCode),
188 'versionName': build.versionName,
189 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
194 if apk['versionCode'] == int(build.versionCode):
199 apklist.append({'versionCode': int(build.versionCode),
200 'versionName': build.versionName,
201 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
203 if app.CurrentVersionCode == '0':
205 # Sort with most recent first...
206 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
208 wikidata += "=Versions=\n"
209 if len(apklist) == 0:
210 wikidata += "We currently have no versions of this app available."
211 elif not gotcurrentver:
212 wikidata += "We don't have the current version of this app."
214 wikidata += "We have the current version of this app."
215 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
216 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
217 if len(app.NoSourceSince) > 0:
218 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
219 if len(app.CurrentVersion) > 0:
220 wikidata += "The current (recommended) version is " + app.CurrentVersion
221 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
224 wikidata += "==" + apk['versionName'] + "==\n"
226 if 'buildproblem' in apk:
227 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
230 wikidata += "This version is built and signed by "
232 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
234 wikidata += "the original developer.\n\n"
235 wikidata += "Version code: " + str(apk['versionCode']) + '\n'
237 wikidata += '\n[[Category:' + wikicat + ']]\n'
238 if len(app.NoSourceSince) > 0:
239 wikidata += '\n[[Category:Apps missing source code]]\n'
240 if validapks == 0 and not app.Disabled:
241 wikidata += '\n[[Category:Apps with no packages]]\n'
242 if cantupdate and not app.Disabled:
243 wikidata += "\n[[Category:Apps we cannot update]]\n"
244 if buildfails and not app.Disabled:
245 wikidata += "\n[[Category:Apps with failing builds]]\n"
246 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
247 wikidata += '\n[[Category:Apps to Update]]\n'
249 wikidata += '\n[[Category:Apps that are disabled]]\n'
250 if app.UpdateCheckMode == 'None' and not app.Disabled:
251 wikidata += '\n[[Category:Apps with no update check]]\n'
252 for appcat in app.Categories:
253 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
255 # We can't have underscores in the page name, even if they're in
256 # the package ID, because MediaWiki messes with them...
257 pagename = appid.replace('_', ' ')
259 # Drop a trailing newline, because mediawiki is going to drop it anyway
260 # and it we don't we'll think the page has changed when it hasn't...
261 if wikidata.endswith('\n'):
262 wikidata = wikidata[:-1]
264 generated_pages[pagename] = wikidata
266 # Make a redirect from the name to the ID too, unless there's
267 # already an existing page with the name and it isn't a redirect.
269 apppagename = app.Name.replace('_', ' ')
270 apppagename = apppagename.replace('{', '')
271 apppagename = apppagename.replace('}', ' ')
272 apppagename = apppagename.replace(':', ' ')
273 apppagename = apppagename.replace('[', ' ')
274 apppagename = apppagename.replace(']', ' ')
275 # Drop double spaces caused mostly by replacing ':' above
276 apppagename = apppagename.replace(' ', ' ')
277 for expagename in site.allpages(prefix=apppagename,
278 filterredir='nonredirects',
280 if expagename == apppagename:
282 # Another reason not to make the redirect page is if the app name
283 # is the same as it's ID, because that will overwrite the real page
284 # with an redirect to itself! (Although it seems like an odd
285 # scenario this happens a lot, e.g. where there is metadata but no
286 # builds or binaries to extract a name from.
287 if apppagename == pagename:
290 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
292 for tcat, genp in [(wikicat, generated_pages),
293 (wikiredircat, generated_redirects)]:
294 catpages = site.Pages['Category:' + tcat]
296 for page in catpages:
297 existingpages.append(page.name)
298 if page.name in genp:
299 pagetxt = page.edit()
300 if pagetxt != genp[page.name]:
301 logging.debug("Updating modified page " + page.name)
302 page.save(genp[page.name], summary='Auto-updated')
304 logging.debug("Page " + page.name + " is unchanged")
306 logging.warn("Deleting page " + page.name)
307 page.delete('No longer published')
308 for pagename, text in genp.items():
309 logging.debug("Checking " + pagename)
310 if pagename not in existingpages:
311 logging.debug("Creating page " + pagename)
313 newpage = site.Pages[pagename]
314 newpage.save(text, summary='Auto-created')
315 except Exception as e:
316 logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
318 # Purge server cache to ensure counts are up to date
319 site.pages['Repository Maintenance'].purge()
322 def delete_disabled_builds(apps, apkcache, repodirs):
323 """Delete disabled build outputs.
325 :param apps: list of all applications, as per metadata.read_metadata
326 :param apkcache: current apk cache information
327 :param repodirs: the repo directories to process
329 for appid, app in apps.items():
330 for build in app['builds']:
331 if not build.disable:
333 apkfilename = common.get_release_filename(app, build)
334 iconfilename = "%s.%s.png" % (
337 for repodir in repodirs:
339 os.path.join(repodir, apkfilename),
340 os.path.join(repodir, apkfilename + '.asc'),
341 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
343 for density in all_screen_densities:
344 repo_dir = get_icon_dir(repodir, density)
345 files.append(os.path.join(repo_dir, iconfilename))
348 if os.path.exists(f):
349 logging.info("Deleting disabled build output " + f)
351 if apkfilename in apkcache:
352 del apkcache[apkfilename]
355 def resize_icon(iconpath, density):
357 if not os.path.isfile(iconpath):
362 fp = open(iconpath, 'rb')
364 size = dpi_to_px(density)
366 if any(length > size for length in im.size):
368 im.thumbnail((size, size), Image.ANTIALIAS)
369 logging.debug("%s was too large at %s - new size is %s" % (
370 iconpath, oldsize, im.size))
371 im.save(iconpath, "PNG")
373 except Exception as e:
374 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
381 def resize_all_icons(repodirs):
382 """Resize all icons that exceed the max size
384 :param repodirs: the repo directories to process
386 for repodir in repodirs:
387 for density in screen_densities:
388 icon_dir = get_icon_dir(repodir, density)
389 icon_glob = os.path.join(icon_dir, '*.png')
390 for iconpath in glob.glob(icon_glob):
391 resize_icon(iconpath, density)
395 """ Get the signing certificate of an apk. To get the same md5 has that
396 Android gets, we encode the .RSA certificate in a specific format and pass
397 it hex-encoded to the md5 digest algorithm.
399 :param apkpath: path to the apk
400 :returns: A string containing the md5 of the signature of the apk or None
401 if an error occurred.
404 with zipfile.ZipFile(apkpath, 'r') as apk:
405 certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
408 logging.error("Found no signing certificates on %s" % apkpath)
411 logging.error("Found multiple signing certificates on %s" % apkpath)
414 cert = apk.read(certs[0])
416 cert_encoded = common.get_certificate(cert)
418 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
421 def get_cache_file():
422 return os.path.join('tmp', 'apkcache')
427 Gather information about all the apk files in the repo directory,
428 using cached data if possible.
431 apkcachefile = get_cache_file()
432 if not options.clean and os.path.exists(apkcachefile):
433 with open(apkcachefile, 'rb') as cf:
434 apkcache = pickle.load(cf, encoding='utf-8')
435 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
443 def write_cache(apkcache):
444 apkcachefile = get_cache_file()
445 cache_path = os.path.dirname(apkcachefile)
446 if not os.path.exists(cache_path):
447 os.makedirs(cache_path)
448 apkcache["METADATA_VERSION"] = METADATA_VERSION
449 with open(apkcachefile, 'wb') as cf:
450 pickle.dump(apkcache, cf)
453 def get_icon_bytes(apkzip, iconsrc):
454 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
456 return apkzip.read(iconsrc)
458 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
461 def sha256sum(filename):
462 '''Calculate the sha256 of the given file'''
463 sha = hashlib.sha256()
464 with open(filename, 'rb') as f:
470 return sha.hexdigest()
473 def has_known_vulnerability(filename):
474 """checks for known vulnerabilities in the APK
476 Checks OpenSSL .so files in the APK to see if they are a known vulnerable
477 version. Google also enforces this:
478 https://support.google.com/faqs/answer/6376725?hl=en
480 Checks whether there are more than one classes.dex or AndroidManifest.xml
481 files, which is invalid and an essential part of the "Master Key" attack.
483 http://www.saurik.com/id/17
486 # statically load this pattern
487 if not hasattr(has_known_vulnerability, "pattern"):
488 has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
491 with zipfile.ZipFile(filename) as zf:
492 for name in zf.namelist():
493 if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
496 chunk = lib.read(4096)
499 m = has_known_vulnerability.pattern.search(chunk)
501 version = m.group(1).decode('ascii')
502 if version.startswith('1.0.1') and version[5] >= 'r' \
503 or version.startswith('1.0.2') and version[5] >= 'f':
504 logging.debug('"%s" contains recent %s (%s)', filename, name, version)
506 logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
509 elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
510 if name in files_in_apk:
512 files_in_apk.add(name)
517 def insert_obbs(repodir, apps, apks):
518 """Scans the .obb files in a given repo directory and adds them to the
519 relevant APK instances. OBB files have versionCodes like APK
520 files, and they are loosely associated. If there is an OBB file
521 present, then any APK with the same or higher versionCode will use
522 that OBB file. There are two OBB types: main and patch, each APK
523 can only have only have one of each.
525 https://developer.android.com/google/play/expansion-files.html
527 :param repodir: repo directory to scan
528 :param apps: list of current, valid apps
529 :param apks: current information on all APKs
533 def obbWarnDelete(f, msg):
534 logging.warning(msg + f)
535 if options.delete_unknown:
536 logging.error("Deleting unknown file: " + f)
540 java_Integer_MIN_VALUE = -pow(2, 31)
541 currentPackageNames = apps.keys()
542 for f in glob.glob(os.path.join(repodir, '*.obb')):
543 obbfile = os.path.basename(f)
544 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
545 chunks = obbfile.split('.')
546 if chunks[0] != 'main' and chunks[0] != 'patch':
547 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
549 if not re.match(r'^-?[0-9]+$', chunks[1]):
550 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
552 versionCode = int(chunks[1])
553 packagename = ".".join(chunks[2:-1])
555 highestVersionCode = java_Integer_MIN_VALUE
556 if packagename not in currentPackageNames:
557 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
560 if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
561 highestVersionCode = apk['versionCode']
562 if versionCode > highestVersionCode:
563 obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
564 + ') than any APK: ')
566 obbsha256 = sha256sum(f)
567 obbs.append((packagename, versionCode, obbfile, obbsha256))
570 for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
571 if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
572 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
573 apk['obbMainFile'] = obbfile
574 apk['obbMainFileSha256'] = obbsha256
575 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
576 apk['obbPatchFile'] = obbfile
577 apk['obbPatchFileSha256'] = obbsha256
578 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
582 def _get_localized_dict(app, locale):
583 '''get the dict to add localized store metadata to'''
584 if 'localized' not in app:
585 app['localized'] = collections.OrderedDict()
586 if locale not in app['localized']:
587 app['localized'][locale] = collections.OrderedDict()
588 return app['localized'][locale]
591 def _set_localized_text_entry(app, locale, key, f):
592 limit = config['char_limits'][key]
593 localized = _get_localized_dict(app, locale)
595 text = fp.read()[:limit]
597 localized[key] = text
600 def _set_author_entry(app, key, f):
601 limit = config['char_limits']['author']
603 text = fp.read()[:limit]
608 def copy_triple_t_store_metadata(apps):
609 """Include store metadata from the app's source repo
611 The Triple-T Gradle Play Publisher is a plugin that has a standard
612 file layout for all of the metadata and graphics that the Google
613 Play Store accepts. Since F-Droid has the git repo, it can just
614 pluck those files directly. This method reads any text files into
615 the app dict, then copies any graphics into the fdroid repo
618 This needs to be run before insert_localized_app_metadata() so that
619 the graphics files that are copied into the fdroid repo get
622 https://github.com/Triple-T/gradle-play-publisher#upload-images
623 https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
627 if not os.path.isdir('build'):
628 return # nothing to do
630 for packageName, app in apps.items():
631 for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
632 logging.debug('Triple-T Gradle Play Publisher: ' + d)
633 for root, dirs, files in os.walk(d):
634 segments = root.split('/')
635 locale = segments[-2]
637 if f == 'fulldescription':
638 _set_localized_text_entry(app, locale, 'description',
639 os.path.join(root, f))
641 elif f == 'shortdescription':
642 _set_localized_text_entry(app, locale, 'summary',
643 os.path.join(root, f))
646 _set_localized_text_entry(app, locale, 'name',
647 os.path.join(root, f))
650 _set_localized_text_entry(app, locale, 'video',
651 os.path.join(root, f))
653 elif f == 'whatsnew':
654 _set_localized_text_entry(app, segments[-1], 'whatsNew',
655 os.path.join(root, f))
657 elif f == 'contactEmail':
658 _set_author_entry(app, 'authorEmail', os.path.join(root, f))
660 elif f == 'contactPhone':
661 _set_author_entry(app, 'authorPhone', os.path.join(root, f))
663 elif f == 'contactWebsite':
664 _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
667 base, extension = common.get_extension(f)
668 dirname = os.path.basename(root)
669 if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
670 if segments[-2] == 'listing':
671 locale = segments[-3]
673 locale = segments[-2]
674 destdir = os.path.join('repo', packageName, locale)
675 os.makedirs(destdir, mode=0o755, exist_ok=True)
676 sourcefile = os.path.join(root, f)
677 destfile = os.path.join(destdir, dirname + '.' + extension)
678 logging.debug('copying ' + sourcefile + ' ' + destfile)
679 shutil.copy(sourcefile, destfile)
682 def insert_localized_app_metadata(apps):
683 """scans standard locations for graphics and localized text
685 Scans for localized description files, store graphics, and
686 screenshot PNG files in statically defined screenshots directory
687 and adds them to the app metadata. The screenshots and graphic
688 must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
689 and must be in the following layout:
690 # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
692 repo/packageName/locale/featureGraphic.png
693 repo/packageName/locale/phoneScreenshots/1.png
694 repo/packageName/locale/phoneScreenshots/2.png
696 The changelog files must be text files named with the versionCode
697 ending with ".txt" and must be in the following layout:
698 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
700 repo/packageName/locale/changelogs/12345.txt
702 This will scan the each app's source repo then the metadata/ dir
703 for these standard locations of changelog files. If it finds
704 them, they will be added to the dict of all packages, with the
705 versions in the metadata/ folder taking precendence over the what
706 is in the app's source repo.
708 Where "packageName" is the app's packageName and "locale" is the locale
709 of the graphics, e.g. what language they are in, using the IETF RFC5646
710 format (en-US, fr-CA, es-MX, etc).
712 This will also scan the app's git for a fastlane folder, and the
713 metadata/ folder and the apps' source repos for standard locations
714 of graphic and screenshot files. If it finds them, it will copy
715 them into the repo. The fastlane files follow this pattern:
716 https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
720 sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
721 sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
722 sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
724 for srcd in sorted(sourcedirs):
725 if not os.path.isdir(srcd):
727 for root, dirs, files in os.walk(srcd):
728 segments = root.split('/')
729 packageName = segments[1]
730 if packageName not in apps:
731 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
733 locale = segments[-1]
734 destdir = os.path.join('repo', packageName, locale)
736 if f in ('description.txt', 'full_description.txt'):
737 _set_localized_text_entry(apps[packageName], locale, 'description',
738 os.path.join(root, f))
740 elif f in ('summary.txt', 'short_description.txt'):
741 _set_localized_text_entry(apps[packageName], locale, 'summary',
742 os.path.join(root, f))
744 elif f in ('name.txt', 'title.txt'):
745 _set_localized_text_entry(apps[packageName], locale, 'name',
746 os.path.join(root, f))
748 elif f == 'video.txt':
749 _set_localized_text_entry(apps[packageName], locale, 'video',
750 os.path.join(root, f))
752 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
753 locale = segments[-2]
754 _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
755 os.path.join(root, f))
758 base, extension = common.get_extension(f)
759 if locale == 'images':
760 locale = segments[-2]
761 destdir = os.path.join('repo', packageName, locale)
762 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
763 os.makedirs(destdir, mode=0o755, exist_ok=True)
764 logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
765 shutil.copy(os.path.join(root, f), destdir)
767 if d in SCREENSHOT_DIRS:
768 for f in glob.glob(os.path.join(root, d, '*.*')):
769 _, extension = common.get_extension(f)
770 if extension in ALLOWED_EXTENSIONS:
771 screenshotdestdir = os.path.join(destdir, d)
772 os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
773 logging.debug('copying ' + f + ' ' + screenshotdestdir)
774 shutil.copy(f, screenshotdestdir)
776 repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
778 if not os.path.isdir(d):
780 for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
781 if not os.path.isfile(f):
783 segments = f.split('/')
784 packageName = segments[1]
786 screenshotdir = segments[3]
787 filename = os.path.basename(f)
788 base, extension = common.get_extension(filename)
790 if packageName not in apps:
791 logging.warning('Found "%s" graphic without metadata for app "%s"!'
792 % (filename, packageName))
794 graphics = _get_localized_dict(apps[packageName], locale)
796 if extension not in ALLOWED_EXTENSIONS:
797 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
798 elif base in GRAPHIC_NAMES:
799 # there can only be zero or one of these per locale
800 graphics[base] = filename
801 elif screenshotdir in SCREENSHOT_DIRS:
802 # there can any number of these per locale
803 logging.debug('adding to ' + screenshotdir + ': ' + f)
804 if screenshotdir not in graphics:
805 graphics[screenshotdir] = []
806 graphics[screenshotdir].append(filename)
808 logging.warning('Unsupported graphics file found: ' + f)
811 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
812 """Scan a repo for all files with an extension except APK/OBB
814 :param apkcache: current cached info about all repo files
815 :param repodir: repo directory to scan
816 :param knownapks: list of all known files, as per metadata.read_metadata
817 :param use_date_from_file: use date from file (instead of current date)
818 for newly added files
823 repodir = repodir.encode('utf-8')
824 for name in os.listdir(repodir):
825 file_extension = common.get_file_extension(name)
826 if file_extension == 'apk' or file_extension == 'obb':
828 filename = os.path.join(repodir, name)
829 name_utf8 = name.decode('utf-8')
830 if filename.endswith(b'_src.tar.gz'):
831 logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
833 if not common.is_repo_file(filename):
835 stat = os.stat(filename)
836 if stat.st_size == 0:
837 raise FDroidException(filename + ' is zero size!')
839 shasum = sha256sum(filename)
842 repo_file = apkcache[name]
843 # added time is cached as tuple but used here as datetime instance
844 if 'added' in repo_file:
845 a = repo_file['added']
846 if isinstance(a, datetime):
847 repo_file['added'] = a
849 repo_file['added'] = datetime(*a[:6])
850 if repo_file.get('hash') == shasum:
851 logging.debug("Reading " + name_utf8 + " from cache")
854 logging.debug("Ignoring stale cache data for " + name)
857 logging.debug("Processing " + name_utf8)
858 repo_file = collections.OrderedDict()
859 repo_file['name'] = os.path.splitext(name_utf8)[0]
860 # TODO rename apkname globally to something more generic
861 repo_file['apkName'] = name_utf8
862 repo_file['hash'] = shasum
863 repo_file['hashType'] = 'sha256'
864 repo_file['versionCode'] = 0
865 repo_file['versionName'] = shasum
866 # the static ID is the SHA256 unless it is set in the metadata
867 repo_file['packageName'] = shasum
869 m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
871 repo_file['packageName'] = m.group(1)
872 repo_file['versionCode'] = int(m.group(2))
873 srcfilename = name + b'_src.tar.gz'
874 if os.path.exists(os.path.join(repodir, srcfilename)):
875 repo_file['srcname'] = srcfilename.decode('utf-8')
876 repo_file['size'] = stat.st_size
878 apkcache[name] = repo_file
881 if use_date_from_file:
882 timestamp = stat.st_ctime
883 default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
885 default_date_param = None
887 # Record in knownapks, getting the added date at the same time..
888 added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
889 default_date=default_date_param)
891 repo_file['added'] = added
893 repo_files.append(repo_file)
895 return repo_files, cachechanged
898 def scan_apk_aapt(apk, apkfile):
899 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
900 if p.returncode != 0:
901 if options.delete_unknown:
902 if os.path.exists(apkfile):
903 logging.error("Failed to get apk information, deleting " + apkfile)
906 logging.error("Could not find {0} to remove it".format(apkfile))
908 logging.error("Failed to get apk information, skipping " + apkfile)
909 raise BuildException("Invalid APK")
910 for line in p.output.splitlines():
911 if line.startswith("package:"):
913 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
914 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
915 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
916 except Exception as e:
917 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
918 elif line.startswith("application:"):
919 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
920 # Keep path to non-dpi icon in case we need it
921 match = re.match(APK_ICON_PAT_NODPI, line)
923 apk['icons_src']['-1'] = match.group(1)
924 elif line.startswith("launchable-activity:"):
925 # Only use launchable-activity as fallback to application
927 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
928 if '-1' not in apk['icons_src']:
929 match = re.match(APK_ICON_PAT_NODPI, line)
931 apk['icons_src']['-1'] = match.group(1)
932 elif line.startswith("application-icon-"):
933 match = re.match(APK_ICON_PAT, line)
935 density = match.group(1)
936 path = match.group(2)
937 apk['icons_src'][density] = path
938 elif line.startswith("sdkVersion:"):
939 m = re.match(APK_SDK_VERSION_PAT, line)
941 logging.error(line.replace('sdkVersion:', '')
942 + ' is not a valid minSdkVersion!')
944 apk['minSdkVersion'] = m.group(1)
945 # if target not set, default to min
946 if 'targetSdkVersion' not in apk:
947 apk['targetSdkVersion'] = m.group(1)
948 elif line.startswith("targetSdkVersion:"):
949 m = re.match(APK_SDK_VERSION_PAT, line)
951 logging.error(line.replace('targetSdkVersion:', '')
952 + ' is not a valid targetSdkVersion!')
954 apk['targetSdkVersion'] = m.group(1)
955 elif line.startswith("maxSdkVersion:"):
956 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
957 elif line.startswith("native-code:"):
958 apk['nativecode'] = []
959 for arch in line[13:].split(' '):
960 apk['nativecode'].append(arch[1:-1])
961 elif line.startswith('uses-permission:'):
962 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
963 if perm_match['maxSdkVersion']:
964 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
965 permission = UsesPermission(
967 perm_match['maxSdkVersion']
970 apk['uses-permission'].append(permission)
971 elif line.startswith('uses-permission-sdk-23:'):
972 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
973 if perm_match['maxSdkVersion']:
974 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
975 permission_sdk_23 = UsesPermissionSdk23(
977 perm_match['maxSdkVersion']
980 apk['uses-permission-sdk-23'].append(permission_sdk_23)
982 elif line.startswith('uses-feature:'):
983 feature = re.match(APK_FEATURE_PAT, line).group(1)
984 # Filter out this, it's only added with the latest SDK tools and
985 # causes problems for lots of apps.
986 if feature != "android.hardware.screen.portrait" \
987 and feature != "android.hardware.screen.landscape":
988 if feature.startswith("android.feature."):
989 feature = feature[16:]
990 apk['features'].add(feature)
993 def scan_apk_androguard(apk, apkfile):
995 from androguard.core.bytecodes.apk import APK
996 apkobject = APK(apkfile)
997 if apkobject.is_valid_APK():
998 arsc = apkobject.get_android_resources()
1000 if options.delete_unknown:
1001 if os.path.exists(apkfile):
1002 logging.error("Failed to get apk information, deleting " + apkfile)
1005 logging.error("Could not find {0} to remove it".format(apkfile))
1007 logging.error("Failed to get apk information, skipping " + apkfile)
1008 raise BuildException("Invaild APK")
1010 raise FDroidException("androguard library is not installed and aapt not present")
1011 except FileNotFoundError:
1012 logging.error("Could not open apk file for analysis")
1013 raise BuildException("Invalid APK")
1015 apk['packageName'] = apkobject.get_package()
1016 apk['versionCode'] = int(apkobject.get_androidversion_code())
1017 apk['versionName'] = apkobject.get_androidversion_name()
1018 if apk['versionName'][0] == "@":
1019 version_id = int(apk['versionName'].replace("@", "0x"), 16)
1020 version_id = arsc.get_id(apk['packageName'], version_id)[1]
1021 apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1022 apk['name'] = apkobject.get_app_name()
1024 if apkobject.get_max_sdk_version() is not None:
1025 apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1026 apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1027 apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1029 icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1030 icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1032 density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1034 for file in apkobject.get_files():
1035 d_re = density_re.match(file)
1037 folder = d_re.group(1).split('-')
1039 resolution = folder[1]
1042 density = screen_resolutions[resolution]
1043 apk['icons_src'][density] = d_re.group(0)
1045 if apk['icons_src'].get('-1') is None:
1046 apk['icons_src']['-1'] = apk['icons_src']['160']
1048 arch_re = re.compile("^lib/(.*)/.*$")
1049 arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1051 apk['nativecode'] = []
1052 apk['nativecode'].extend(sorted(list(arch)))
1054 xml = apkobject.get_android_manifest_xml()
1056 for item in xml.getElementsByTagName('uses-permission'):
1057 name = str(item.getAttribute("android:name"))
1058 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1059 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1060 permission = UsesPermission(
1064 apk['uses-permission'].append(permission)
1066 for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1067 name = str(item.getAttribute("android:name"))
1068 maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1069 maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1070 permission_sdk_23 = UsesPermissionSdk23(
1074 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1076 for item in xml.getElementsByTagName('uses-feature'):
1077 feature = str(item.getAttribute("android:name"))
1078 if feature != "android.hardware.screen.portrait" \
1079 and feature != "android.hardware.screen.landscape":
1080 if feature.startswith("android.feature."):
1081 feature = feature[16:]
1082 apk['features'].append(feature)
1085 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1086 """Scan the apk with the given filename in the given repo directory.
1088 This also extracts the icons.
1090 :param apkcache: current apk cache information
1091 :param apkfilename: the filename of the apk to scan
1092 :param repodir: repo directory to scan
1093 :param knownapks: known apks info
1094 :param use_date_from_apk: use date from APK (instead of current date)
1095 for newly added APKs
1096 :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1097 apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1100 if ' ' in apkfilename:
1101 if options.rename_apks:
1102 newfilename = apkfilename.replace(' ', '_')
1103 os.rename(os.path.join(repodir, apkfilename),
1104 os.path.join(repodir, newfilename))
1105 apkfilename = newfilename
1107 logging.critical("Spaces in filenames are not allowed.")
1108 return True, None, False
1110 apkfile = os.path.join(repodir, apkfilename)
1111 shasum = sha256sum(apkfile)
1113 cachechanged = False
1115 if apkfilename in apkcache:
1116 apk = apkcache[apkfilename]
1117 if apk.get('hash') == shasum:
1118 logging.debug("Reading " + apkfilename + " from cache")
1121 logging.debug("Ignoring stale cache data for " + apkfilename)
1124 logging.debug("Processing " + apkfilename)
1126 apk['hash'] = shasum
1127 apk['hashType'] = 'sha256'
1128 apk['uses-permission'] = []
1129 apk['uses-permission-sdk-23'] = []
1130 apk['features'] = []
1131 apk['icons_src'] = {}
1133 apk['antiFeatures'] = set()
1136 if SdkToolsPopen(['aapt', 'version'], output=False):
1137 scan_apk_aapt(apk, apkfile)
1139 scan_apk_androguard(apk, apkfile)
1140 except BuildException:
1141 return True, None, False
1143 if 'minSdkVersion' not in apk:
1144 logging.warn("No SDK version information found in {0}".format(apkfile))
1145 apk['minSdkVersion'] = 1
1147 # Check for debuggable apks...
1148 if common.isApkAndDebuggable(apkfile):
1149 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1151 # Get the signature (or md5 of, to be precise)...
1152 logging.debug('Getting signature of {0}'.format(apkfile))
1153 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1155 logging.critical("Failed to get apk signature")
1156 return True, None, False
1158 if options.rename_apks:
1159 n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1160 std_short_name = os.path.join(repodir, n)
1161 if apkfile != std_short_name:
1162 if os.path.exists(std_short_name):
1163 std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1164 if apkfile != std_long_name:
1165 if os.path.exists(std_long_name):
1166 dupdir = os.path.join('duplicates', repodir)
1167 if not os.path.isdir(dupdir):
1168 os.makedirs(dupdir, exist_ok=True)
1169 dupfile = os.path.join('duplicates', std_long_name)
1170 logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1171 os.rename(apkfile, dupfile)
1172 return True, None, False
1174 os.rename(apkfile, std_long_name)
1175 apkfile = std_long_name
1177 os.rename(apkfile, std_short_name)
1178 apkfile = std_short_name
1179 apkfilename = apkfile[len(repodir) + 1:]
1181 apk['apkName'] = apkfilename
1182 srcfilename = apkfilename[:-4] + "_src.tar.gz"
1183 if os.path.exists(os.path.join(repodir, srcfilename)):
1184 apk['srcname'] = srcfilename
1185 apk['size'] = os.path.getsize(apkfile)
1187 # verify the jar signature is correct
1188 if not common.verify_apk_signature(apkfile):
1189 return True, None, False
1191 if has_known_vulnerability(apkfile):
1192 apk['antiFeatures'].add('KnownVuln')
1194 apkzip = zipfile.ZipFile(apkfile, 'r')
1196 # if an APK has files newer than the system time, suggest updating
1197 # the system clock. This is useful for offline systems, used for
1198 # signing, which do not have another source of clock sync info. It
1199 # has to be more than 24 hours newer because ZIP/APK files do not
1200 # store timezone info
1201 manifest = apkzip.getinfo('AndroidManifest.xml')
1202 if manifest.date_time[1] == 0: # month can't be zero
1203 logging.debug('AndroidManifest.xml has no date')
1205 dt_obj = datetime(*manifest.date_time)
1206 checkdt = dt_obj - timedelta(1)
1207 if datetime.today() < checkdt:
1208 logging.warn('System clock is older than manifest in: '
1210 + '\nSet clock to that time using:\n'
1211 + 'sudo date -s "' + str(dt_obj) + '"')
1213 iconfilename = "%s.%s.png" % (
1217 # Extract the icon file...
1218 empty_densities = []
1219 for density in screen_densities:
1220 if density not in apk['icons_src']:
1221 empty_densities.append(density)
1223 iconsrc = apk['icons_src'][density]
1224 icon_dir = get_icon_dir(repodir, density)
1225 icondest = os.path.join(icon_dir, iconfilename)
1228 with open(icondest, 'wb') as f:
1229 f.write(get_icon_bytes(apkzip, iconsrc))
1230 apk['icons'][density] = iconfilename
1231 except (zipfile.BadZipFile, ValueError, KeyError) as e:
1232 logging.warning("Error retrieving icon file: %s" % (icondest))
1233 del apk['icons_src'][density]
1234 empty_densities.append(density)
1236 if '-1' in apk['icons_src']:
1237 iconsrc = apk['icons_src']['-1']
1238 iconpath = os.path.join(
1239 get_icon_dir(repodir, '0'), iconfilename)
1240 with open(iconpath, 'wb') as f:
1241 f.write(get_icon_bytes(apkzip, iconsrc))
1243 im = Image.open(iconpath)
1244 dpi = px_to_dpi(im.size[0])
1245 for density in screen_densities:
1246 if density in apk['icons']:
1248 if density == screen_densities[-1] or dpi >= int(density):
1249 apk['icons'][density] = iconfilename
1250 shutil.move(iconpath,
1251 os.path.join(get_icon_dir(repodir, density), iconfilename))
1252 empty_densities.remove(density)
1254 except Exception as e:
1255 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1258 apk['icon'] = iconfilename
1262 # First try resizing down to not lose quality
1264 for density in screen_densities:
1265 if density not in empty_densities:
1266 last_density = density
1268 if last_density is None:
1270 logging.debug("Density %s not available, resizing down from %s"
1271 % (density, last_density))
1273 last_iconpath = os.path.join(
1274 get_icon_dir(repodir, last_density), iconfilename)
1275 iconpath = os.path.join(
1276 get_icon_dir(repodir, density), iconfilename)
1279 fp = open(last_iconpath, 'rb')
1282 size = dpi_to_px(density)
1284 im.thumbnail((size, size), Image.ANTIALIAS)
1285 im.save(iconpath, "PNG")
1286 empty_densities.remove(density)
1287 except Exception as e:
1288 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1293 # Then just copy from the highest resolution available
1295 for density in reversed(screen_densities):
1296 if density not in empty_densities:
1297 last_density = density
1299 if last_density is None:
1301 logging.debug("Density %s not available, copying from lower density %s"
1302 % (density, last_density))
1305 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1306 os.path.join(get_icon_dir(repodir, density), iconfilename))
1308 empty_densities.remove(density)
1310 for density in screen_densities:
1311 icon_dir = get_icon_dir(repodir, density)
1312 icondest = os.path.join(icon_dir, iconfilename)
1313 resize_icon(icondest, density)
1315 # Copy from icons-mdpi to icons since mdpi is the baseline density
1316 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1317 if os.path.isfile(baseline):
1318 apk['icons']['0'] = iconfilename
1319 shutil.copyfile(baseline,
1320 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1322 if use_date_from_apk and manifest.date_time[1] != 0:
1323 default_date_param = datetime(*manifest.date_time)
1325 default_date_param = None
1327 # Record in known apks, getting the added date at the same time..
1328 added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1329 default_date=default_date_param)
1331 apk['added'] = added
1333 apkcache[apkfilename] = apk
1336 return False, apk, cachechanged
1339 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1340 """Scan the apks in the given repo directory.
1342 This also extracts the icons.
1344 :param apkcache: current apk cache information
1345 :param repodir: repo directory to scan
1346 :param knownapks: known apks info
1347 :param use_date_from_apk: use date from APK (instead of current date)
1348 for newly added APKs
1349 :returns: (apks, cachechanged) where apks is a list of apk information,
1350 and cachechanged is True if the apkcache got changed.
1353 cachechanged = False
1355 for icon_dir in get_all_icon_dirs(repodir):
1356 if os.path.exists(icon_dir):
1358 shutil.rmtree(icon_dir)
1359 os.makedirs(icon_dir)
1361 os.makedirs(icon_dir)
1364 for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1365 apkfilename = apkfile[len(repodir) + 1:]
1366 (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1371 return apks, cachechanged
1374 def apply_info_from_latest_apk(apps, apks):
1376 Some information from the apks needs to be applied up to the application level.
1377 When doing this, we use the info from the most recent version's apk.
1378 We deal with figuring out when the app was added and last updated at the same time.
1380 for appid, app in apps.items():
1381 bestver = UNSET_VERSION_CODE
1383 if apk['packageName'] == appid:
1384 if apk['versionCode'] > bestver:
1385 bestver = apk['versionCode']
1389 if not app.added or apk['added'] < app.added:
1390 app.added = apk['added']
1391 if not app.lastUpdated or apk['added'] > app.lastUpdated:
1392 app.lastUpdated = apk['added']
1395 logging.debug("Don't know when " + appid + " was added")
1396 if not app.lastUpdated:
1397 logging.debug("Don't know when " + appid + " was last updated")
1399 if bestver == UNSET_VERSION_CODE:
1401 if app.Name is None:
1402 app.Name = app.AutoName or appid
1404 logging.debug("Application " + appid + " has no packages")
1406 if app.Name is None:
1407 app.Name = bestapk['name']
1408 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1409 if app.CurrentVersionCode is None:
1410 app.CurrentVersionCode = str(bestver)
1413 def make_categories_txt(repodir, categories):
1414 '''Write a category list in the repo to allow quick access'''
1416 for cat in sorted(categories):
1417 catdata += cat + '\n'
1418 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1422 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1424 def move_file(from_dir, to_dir, filename, ignore_missing):
1425 from_path = os.path.join(from_dir, filename)
1426 if ignore_missing and not os.path.exists(from_path):
1428 to_path = os.path.join(to_dir, filename)
1429 shutil.move(from_path, to_path)
1431 def filter_apk_list_sorted(apk_list):
1433 for apk in apk_list:
1434 if apk['packageName'] == appid:
1437 # Sort the apk list by version code. First is highest/newest.
1438 return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1440 for appid, app in apps.items():
1442 if app.ArchivePolicy:
1443 keepversions = int(app.ArchivePolicy[:-9])
1445 keepversions = defaultkeepversions
1447 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1448 .format(appid, len(apks), keepversions, len(archapks)))
1450 current_app_apks = filter_apk_list_sorted(apks)
1451 if len(current_app_apks) > keepversions:
1452 # Move back the ones we don't want.
1453 for apk in current_app_apks[keepversions:]:
1454 logging.info("Moving " + apk['apkName'] + " to archive")
1455 move_file(repodir, archivedir, apk['apkName'], False)
1456 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1457 for density in all_screen_densities:
1458 repo_icon_dir = get_icon_dir(repodir, density)
1459 archive_icon_dir = get_icon_dir(archivedir, density)
1460 if density not in apk['icons']:
1462 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1463 if 'srcname' in apk:
1464 move_file(repodir, archivedir, apk['srcname'], False)
1465 archapks.append(apk)
1468 current_app_archapks = filter_apk_list_sorted(archapks)
1469 if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1470 required = keepversions - len(apks)
1471 # Move forward the ones we want again.
1472 for apk in current_app_archapks[:required]:
1473 logging.info("Moving " + apk['apkName'] + " from archive")
1474 move_file(archivedir, repodir, apk['apkName'], False)
1475 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1476 for density in all_screen_densities:
1477 repo_icon_dir = get_icon_dir(repodir, density)
1478 archive_icon_dir = get_icon_dir(archivedir, density)
1479 if density not in apk['icons']:
1481 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1482 if 'srcname' in apk:
1483 move_file(archivedir, repodir, apk['srcname'], False)
1484 archapks.remove(apk)
1488 def add_apks_to_per_app_repos(repodir, apks):
1489 apks_per_app = dict()
1491 apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1492 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1493 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1494 apks_per_app[apk['packageName']] = apk
1496 if not os.path.exists(apk['per_app_icons']):
1497 logging.info('Adding new repo for only ' + apk['packageName'])
1498 os.makedirs(apk['per_app_icons'])
1500 apkpath = os.path.join(repodir, apk['apkName'])
1501 shutil.copy(apkpath, apk['per_app_repo'])
1502 apksigpath = apkpath + '.sig'
1503 if os.path.exists(apksigpath):
1504 shutil.copy(apksigpath, apk['per_app_repo'])
1505 apkascpath = apkpath + '.asc'
1506 if os.path.exists(apkascpath):
1507 shutil.copy(apkascpath, apk['per_app_repo'])
1516 global config, options
1518 # Parse command line...
1519 parser = ArgumentParser()
1520 common.setup_global_opts(parser)
1521 parser.add_argument("--create-key", action="store_true", default=False,
1522 help="Create a repo signing key in a keystore")
1523 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1524 help="Create skeleton metadata files that are missing")
1525 parser.add_argument("--delete-unknown", action="store_true", default=False,
1526 help="Delete APKs and/or OBBs without metadata from the repo")
1527 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1528 help="Report on build data status")
1529 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1530 help="Interactively ask about things that need updating.")
1531 parser.add_argument("-I", "--icons", action="store_true", default=False,
1532 help="Resize all the icons exceeding the max pixel size and exit")
1533 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1534 help="Specify editor to use in interactive mode. Default " +
1535 "is /etc/alternatives/editor")
1536 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1537 help="Update the wiki")
1538 parser.add_argument("--pretty", action="store_true", default=False,
1539 help="Produce human-readable index.xml")
1540 parser.add_argument("--clean", action="store_true", default=False,
1541 help="Clean update - don't uses caches, reprocess all apks")
1542 parser.add_argument("--nosign", action="store_true", default=False,
1543 help="When configured for signed indexes, create only unsigned indexes at this stage")
1544 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1545 help="Use date from apk instead of current time for newly added apks")
1546 parser.add_argument("--rename-apks", action="store_true", default=False,
1547 help="Rename APK files that do not match package.name_123.apk")
1548 metadata.add_metadata_arguments(parser)
1549 options = parser.parse_args()
1550 metadata.warnings_action = options.W
1552 config = common.read_config(options)
1554 if not ('jarsigner' in config and 'keytool' in config):
1555 raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1558 if config['archive_older'] != 0:
1559 repodirs.append('archive')
1560 if not os.path.exists('archive'):
1564 resize_all_icons(repodirs)
1567 if options.rename_apks:
1568 options.clean = True
1570 # check that icons exist now, rather than fail at the end of `fdroid update`
1571 for k in ['repo_icon', 'archive_icon']:
1573 if not os.path.exists(config[k]):
1574 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1577 # if the user asks to create a keystore, do it now, reusing whatever it can
1578 if options.create_key:
1579 if os.path.exists(config['keystore']):
1580 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1581 logging.critical("\t'" + config['keystore'] + "'")
1584 if 'repo_keyalias' not in config:
1585 config['repo_keyalias'] = socket.getfqdn()
1586 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1587 if 'keydname' not in config:
1588 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1589 common.write_to_config(config, 'keydname', config['keydname'])
1590 if 'keystore' not in config:
1591 config['keystore'] = common.default_config['keystore']
1592 common.write_to_config(config, 'keystore', config['keystore'])
1594 password = common.genpassword()
1595 if 'keystorepass' not in config:
1596 config['keystorepass'] = password
1597 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1598 if 'keypass' not in config:
1599 config['keypass'] = password
1600 common.write_to_config(config, 'keypass', config['keypass'])
1601 common.genkeystore(config)
1604 apps = metadata.read_metadata()
1606 # Generate a list of categories...
1608 for app in apps.values():
1609 categories.update(app.Categories)
1611 # Read known apks data (will be updated and written back when we've finished)
1612 knownapks = common.KnownApks()
1615 apkcache = get_cache()
1617 # Delete builds for disabled apps
1618 delete_disabled_builds(apps, apkcache, repodirs)
1620 # Scan all apks in the main repo
1621 apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1623 files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1624 options.use_date_from_apk)
1625 cachechanged = cachechanged or fcachechanged
1627 # Generate warnings for apk's with no metadata (or create skeleton
1628 # metadata files, if requested on the command line)
1631 if apk['packageName'] not in apps:
1632 if options.create_metadata:
1633 if 'name' not in apk:
1634 logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1636 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1637 f.write("License:Unknown\n")
1638 f.write("Web Site:\n")
1639 f.write("Source Code:\n")
1640 f.write("Issue Tracker:\n")
1641 f.write("Changelog:\n")
1642 f.write("Summary:" + apk['name'] + "\n")
1643 f.write("Description:\n")
1644 f.write(apk['name'] + "\n")
1646 f.write("Name:" + apk['name'] + "\n")
1648 logging.info("Generated skeleton metadata for " + apk['packageName'])
1651 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1652 if options.delete_unknown:
1653 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1654 rmf = os.path.join(repodirs[0], apk['apkName'])
1655 if not os.path.exists(rmf):
1656 logging.error("Could not find {0} to remove it".format(rmf))
1660 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1662 # update the metadata with the newly created ones included
1664 apps = metadata.read_metadata()
1666 copy_triple_t_store_metadata(apps)
1667 insert_obbs(repodirs[0], apps, apks)
1668 insert_localized_app_metadata(apps)
1670 # Scan the archive repo for apks as well
1671 if len(repodirs) > 1:
1672 archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1678 # Apply information from latest apks to the application and update dates
1679 apply_info_from_latest_apk(apps, apks + archapks)
1681 # Sort the app list by name, then the web site doesn't have to by default.
1682 # (we had to wait until we'd scanned the apks to do this, because mostly the
1683 # name comes from there!)
1684 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1686 # APKs are placed into multiple repos based on the app package, providing
1687 # per-app subscription feeds for nightly builds and things like it
1688 if config['per_app_repos']:
1689 add_apks_to_per_app_repos(repodirs[0], apks)
1690 for appid, app in apps.items():
1691 repodir = os.path.join(appid, 'fdroid', 'repo')
1693 appdict[appid] = app
1694 if os.path.isdir(repodir):
1695 index.make(appdict, [appid], apks, repodir, False)
1697 logging.info('Skipping index generation for ' + appid)
1700 if len(repodirs) > 1:
1701 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1703 # Make the index for the main repo...
1704 index.make(apps, sortedids, apks, repodirs[0], False)
1705 make_categories_txt(repodirs[0], categories)
1707 # If there's an archive repo, make the index for it. We already scanned it
1709 if len(repodirs) > 1:
1710 index.make(apps, sortedids, archapks, repodirs[1], True)
1712 git_remote = config.get('binary_transparency_remote')
1713 if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1715 btlog.make_binary_transparency_log(repodirs)
1717 if config['update_stats']:
1718 # Update known apks info...
1719 knownapks.writeifchanged()
1721 # Generate latest apps data for widget
1722 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1724 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1726 appid = line.rstrip()
1727 data += appid + "\t"
1729 data += app.Name + "\t"
1730 if app.icon is not None:
1731 data += app.icon + "\t"
1732 data += app.License + "\n"
1733 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1737 write_cache(apkcache)
1739 # Update the wiki...
1741 update_wiki(apps, sortedids, apks + archapks)
1743 logging.info("Finished.")
1746 if __name__ == "__main__":