fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         if app['Current Version Code'] == '0':
 145             cantupdate = True
 146         # Sort with most recent first...
 147         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 148
 149         wikidata += "=Versions=\n"
 150         if len(apklist) == 0:
 151             wikidata += "We currently have no versions of this app available."
 152         elif not gotcurrentver:
 153             wikidata += "We don't have the current version of this app."
 154         else:
 155             wikidata += "We have the current version of this app."
 156         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 157         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 158         if len(app['No Source Since']) > 0:
 159             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 160         if len(app['Current Version']) > 0:
 161             wikidata += "The current (recommended) version is " + app['Current Version']
 162             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 163         validapks = 0
 164         for apk in apklist:
 165             wikidata += "==" + apk['version'] + "==\n"
 166
 167             if 'buildproblem' in apk:
 168                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 169             else:
 170                 validapks += 1
 171                 wikidata += "This version is built and signed by "
 172                 if 'srcname' in apk:
 173                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 174                 else:
 175                     wikidata += "the original developer.\n\n"
 176             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 177
 178         wikidata += '\n[[Category:' + wikicat + ']]\n'
 179         if len(app['No Source Since']) > 0:
 180             wikidata += '\n[[Category:Apps missing source code]]\n'
 181         if validapks == 0 and not app['Disabled']:
 182             wikidata += '\n[[Category:Apps with no packages]]\n'
 183         if cantupdate and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps we can't update]]\n"
 185         if buildfails and not app['Disabled']:
 186             wikidata += "\n[[Category:Apps with failing builds]]\n"
 187         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 188             wikidata += '\n[[Category:Apps to Update]]\n'
 189         if app['Disabled']:
 190             wikidata += '\n[[Category:Apps that are disabled]]\n'
 191         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 192             wikidata += '\n[[Category:Apps with no update check]]\n'
 193         for appcat in app['Categories']:
 194             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 195
 196         # We can't have underscores in the page name, even if they're in
 197         # the package ID, because MediaWiki messes with them...
 198         pagename = app['id'].replace('_', ' ')
 199
 200         # Drop a trailing newline, because mediawiki is going to drop it anyway
 201         # and it we don't we'll think the page has changed when it hasn't...
 202         if wikidata.endswith('\n'):
 203             wikidata = wikidata[:-1]
 204
 205         generated_pages[pagename] = wikidata
 206
 207         # Make a redirect from the name to the ID too, unless there's
 208         # already an existing page with the name and it isn't a redirect.
 209         noclobber = False
 210         apppagename = app['Name'].replace('_', ' ')
 211         apppagename = apppagename.replace('{', '')
 212         apppagename = apppagename.replace('}', ' ')
 213         apppagename = apppagename.replace(':', ' ')
 214         # Drop double spaces caused mostly by replacing ':' above
 215         apppagename = apppagename.replace('  ', ' ')
 216         for expagename in site.allpages(prefix=apppagename,
 217                 filterredir='nonredirects', generator=False):
 218             if expagename == apppagename:
 219                 noclobber = True
 220         # Another reason not to make the redirect page is if the app name
 221         # is the same as it's ID, because that will overwrite the real page
 222         # with an redirect to itself! (Although it seems like an odd
 223         # scenario this happens a lot, e.g. where there is metadata but no
 224         # builds or binaries to extract a name from.
 225         if apppagename == pagename:
 226             noclobber = True
 227         if not noclobber:
 228             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 229
 230     for tcat, genp in [(wikicat, generated_pages),
 231             (wikiredircat, generated_redirects)]:
 232         catpages = site.Pages['Category:' + tcat]
 233         existingpages = []
 234         for page in catpages:
 235             existingpages.append(page.name)
 236             if page.name in genp:
 237                 pagetxt = page.edit()
 238                 if pagetxt != genp[page.name]:
 239                     logging.debug("Updating modified page " + page.name)
 240                     page.save(genp[page.name], summary='Auto-updated')
 241                 else:
 242                     logging.debug("Page " + page.name + " is unchanged")
 243             else:
 244                 logging.warn("Deleting page " + page.name)
 245                 page.delete('No longer published')
 246         for pagename, text in genp.items():
 247             logging.debug("Checking " + pagename)
 248             if not pagename in existingpages:
 249                 logging.debug("Creating page " + pagename)
 250                 try:
 251                     newpage = site.Pages[pagename]
 252                     newpage.save(text, summary='Auto-created')
 253                 except:
 254                     logging.error("...FAILED to create page")
 255
 256     # Purge server cache to ensure counts are up to date
 257     site.pages['Repository Maintenance'].purge()
 258
 259 def delete_disabled_builds(apps, apkcache, repodirs):
 260     """Delete disabled build outputs.
 261
 262     :param apps: list of all applications, as per metadata.read_metadata
 263     :param apkcache: current apk cache information
 264     :param repodirs: the repo directories to process
 265     """
 266     for app in apps:
 267         for build in app['builds']:
 268             if 'disable' in build:
 269                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 270                 for repodir in repodirs:
 271                     apkpath = os.path.join(repodir, apkfilename)
 272                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 273                     for name in [apkpath, srcpath]:
 274                         if os.path.exists(name):
 275                             logging.warn("Deleting disabled build output " + apkfilename)
 276                             os.remove(name)
 277                 if apkfilename in apkcache:
 278                     del apkcache[apkfilename]
 279
 280 def resize_icon(iconpath, density):
 281
 282     if not os.path.isfile(iconpath):
 283         return
 284
 285     try:
 286         im = Image.open(iconpath)
 287         size = dpi_to_px(density)
 288
 289         if any(length > size for length in im.size):
 290             oldsize = im.size
 291             im.thumbnail((size, size), Image.ANTIALIAS)
 292             logging.debug("%s was too large at %s - new size is %s" % (
 293                 iconpath, oldsize, im.size))
 294             im.save(iconpath, "PNG")
 295
 296         else:
 297             logging.info("%s is small enough: %s" % im.size)
 298
 299     except Exception, e:
 300         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 301
 302 def resize_all_icons(repodirs):
 303     """Resize all icons that exceed the max size
 304
 305     :param repodirs: the repo directories to process
 306     """
 307     for repodir in repodirs:
 308         for density in get_densities():
 309             icon_dir = get_icon_dir(repodir, density)
 310             icon_glob = os.path.join(icon_dir, '*.png')
 311             for iconpath in glob.glob(icon_glob):
 312                 resize_icon(iconpath, density)
 313
 314 def scan_apks(apps, apkcache, repodir, knownapks):
 315     """Scan the apks in the given repo directory.
 316
 317     This also extracts the icons.
 318
 319     :param apps: list of all applications, as per metadata.read_metadata
 320     :param apkcache: current apk cache information
 321     :param repodir: repo directory to scan
 322     :param knownapks: known apks info
 323     :returns: (apks, cachechanged) where apks is a list of apk information,
 324               and cachechanged is True if the apkcache got changed.
 325     """
 326
 327     cachechanged = False
 328
 329     icon_dirs = get_icon_dirs(repodir)
 330     for icon_dir in icon_dirs:
 331         if os.path.exists(icon_dir):
 332             if options.clean:
 333                 shutil.rmtree(icon_dir)
 334                 os.makedirs(icon_dir)
 335         else:
 336             os.makedirs(icon_dir)
 337
 338     apks = []
 339     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 340     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 341     vername_pat = re.compile(".*versionName='([^']*)'.*")
 342     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 343     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 344     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 345     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 346     string_pat = re.compile(".*'([^']*)'.*")
 347     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 348
 349         apkfilename = apkfile[len(repodir) + 1:]
 350         if ' ' in apkfilename:
 351             logging.error("No spaces in APK filenames!")
 352             sys.exit(1)
 353
 354         if apkfilename in apkcache:
 355             logging.debug("Reading " + apkfilename + " from cache")
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             logging.info("Processing " + apkfilename)
 361             thisinfo = {}
 362             thisinfo['apkname'] = apkfilename
 363             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 364             if os.path.exists(os.path.join(repodir, srcfilename)):
 365                 thisinfo['srcname'] = srcfilename
 366             thisinfo['size'] = os.path.getsize(apkfile)
 367             thisinfo['permissions'] = []
 368             thisinfo['features'] = []
 369             thisinfo['icons_src'] = {}
 370             thisinfo['icons'] = {}
 371             p = FDroidPopen([os.path.join(config['sdk_path'],
 372                         'build-tools', config['build_tools'], 'aapt'),
 373                     'dump', 'badging', apkfile])
 374             if p.returncode != 0:
 375                 logging.critical("Failed to get apk information")
 376                 sys.exit(1)
 377             for line in p.stdout.splitlines():
 378                 if line.startswith("package:"):
 379                     try:
 380                         thisinfo['id'] = re.match(name_pat, line).group(1)
 381                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 382                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 383                     except Exception, e:
 384                         logging.info("Package matching failed: " + str(e))
 385                         logging.info("Line was: " + line)
 386                         sys.exit(1)
 387                 elif line.startswith("application:"):
 388                     thisinfo['name'] = re.match(label_pat, line).group(1)
 389                     # Keep path to non-dpi icon in case we need it
 390                     match = re.match(icon_pat_nodpi, line)
 391                     if match:
 392                         thisinfo['icons_src']['-1'] = match.group(1)
 393                 elif line.startswith("launchable-activity:"):
 394                     # Only use launchable-activity as fallback to application
 395                     if '-1' not in thisinfo['icons_src']:
 396                         match = re.match(icon_pat_nodpi, line)
 397                         if match:
 398                             thisinfo['icons_src']['-1'] = match.group(1)
 399                 elif line.startswith("application-icon-"):
 400                     match = re.match(icon_pat, line)
 401                     if match:
 402                         density = match.group(1)
 403                         path = match.group(2)
 404                         thisinfo['icons_src'][density] = path
 405                 elif line.startswith("sdkVersion:"):
 406                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 407                 elif line.startswith("maxSdkVersion:"):
 408                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 logging.warn("no SDK version information found")
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 450                 sys.exit(1)
 451             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 452                         'getsig', os.path.join(os.getcwd(), apkfile)])
 453             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 454                 logging.critical("Failed to get apk signature")
 455                 sys.exit(1)
 456             thisinfo['sig'] = p.stdout[7:].strip()
 457
 458             apk = zipfile.ZipFile(apkfile, 'r')
 459
 460             iconfilename = "%s.%s.png" % (
 461                     thisinfo['id'],
 462                     thisinfo['versioncode'])
 463
 464             # Extract the icon file...
 465             densities = get_densities()
 466             empty_densities = []
 467             for density in densities:
 468                 if density not in thisinfo['icons_src']:
 469                     empty_densities.append(density)
 470                     continue
 471                 iconsrc = thisinfo['icons_src'][density]
 472                 icon_dir = get_icon_dir(repodir, density)
 473                 icondest = os.path.join(icon_dir, iconfilename)
 474
 475                 try:
 476                     iconfile = open(icondest, 'wb')
 477                     iconfile.write(apk.read(iconsrc))
 478                     iconfile.close()
 479                     thisinfo['icons'][density] = iconfilename
 480
 481                 except:
 482                     logging.warn("Error retrieving icon file")
 483                     del thisinfo['icons'][density]
 484                     del thisinfo['icons_src'][density]
 485                     empty_densities.append(density)
 486
 487             if '-1' in thisinfo['icons_src']:
 488                 iconsrc = thisinfo['icons_src']['-1']
 489                 iconpath = os.path.join(
 490                         get_icon_dir(repodir, None), iconfilename)
 491                 iconfile = open(iconpath, 'wb')
 492                 iconfile.write(apk.read(iconsrc))
 493                 iconfile.close()
 494                 try:
 495                     im = Image.open(iconpath)
 496                     dpi = px_to_dpi(im.size[0])
 497                     for density in densities:
 498                         if density in thisinfo['icons']:
 499                             break
 500                         if density == densities[-1] or dpi >= int(density):
 501                             thisinfo['icons'][density] = iconfilename
 502                             shutil.move(iconpath,
 503                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 504                             empty_densities.remove(density)
 505                             break
 506                 except Exception, e:
 507                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 508
 509             if thisinfo['icons']:
 510                 thisinfo['icon'] = iconfilename
 511
 512             apk.close()
 513
 514             # First try resizing down to not lose quality
 515             last_density = None
 516             for density in densities:
 517                 if density not in empty_densities:
 518                     last_density = density
 519                     continue
 520                 if last_density is None:
 521                     continue
 522                 logging.info("Density %s not available, resizing down from %s" % (
 523                         density, last_density))
 524
 525                 last_iconpath = os.path.join(
 526                         get_icon_dir(repodir, last_density), iconfilename)
 527                 iconpath = os.path.join(
 528                         get_icon_dir(repodir, density), iconfilename)
 529                 try:
 530                     im = Image.open(last_iconpath)
 531                 except:
 532                     logging.warn("Invalid image file at %s" % last_iconpath)
 533                     continue
 534
 535                 size = dpi_to_px(density)
 536
 537                 im.thumbnail((size, size), Image.ANTIALIAS)
 538                 im.save(iconpath, "PNG")
 539                 empty_densities.remove(density)
 540
 541             # Then just copy from the highest resolution available
 542             last_density = None
 543             for density in reversed(densities):
 544                 if density not in empty_densities:
 545                     last_density = density
 546                     continue
 547                 if last_density is None:
 548                     continue
 549                 logging.info("Density %s not available, copying from lower density %s" % (
 550                         density, last_density))
 551
 552                 shutil.copyfile(
 553                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 554                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 555
 556                 empty_densities.remove(density)
 557
 558             for density in densities:
 559                 icon_dir = get_icon_dir(repodir, density)
 560                 icondest = os.path.join(icon_dir, iconfilename)
 561                 resize_icon(icondest, density)
 562
 563             # Copy from icons-mdpi to icons since mdpi is the baseline density
 564             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 565             if os.path.isfile(baseline):
 566                 shutil.copyfile(baseline,
 567                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 568
 569             # Record in known apks, getting the added date at the same time..
 570             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 571             if added:
 572                 thisinfo['added'] = added
 573
 574             apkcache[apkfilename] = thisinfo
 575             cachechanged = True
 576
 577         apks.append(thisinfo)
 578
 579     return apks, cachechanged
 580
 581
 582 repo_pubkey_fingerprint = None
 583
 584 def make_index(apps, apks, repodir, archive, categories):
 585     """Make a repo index.
 586
 587     :param apps: fully populated apps list
 588     :param apks: full populated apks list
 589     :param repodir: the repo directory
 590     :param archive: True if this is the archive repo, False if it's the
 591                     main one.
 592     :param categories: list of categories
 593     """
 594
 595     doc = Document()
 596
 597     def addElement(name, value, doc, parent):
 598         el = doc.createElement(name)
 599         el.appendChild(doc.createTextNode(value))
 600         parent.appendChild(el)
 601     def addElementCDATA(name, value, doc, parent):
 602         el = doc.createElement(name)
 603         el.appendChild(doc.createCDATASection(value))
 604         parent.appendChild(el)
 605
 606     root = doc.createElement("fdroid")
 607     doc.appendChild(root)
 608
 609     repoel = doc.createElement("repo")
 610
 611     if archive:
 612         repoel.setAttribute("name", config['archive_name'])
 613         if config['repo_maxage'] != 0:
 614             repoel.setAttribute("maxage", str(config['repo_maxage']))
 615         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 616         repoel.setAttribute("url", config['archive_url'])
 617         addElement('description', config['archive_description'], doc, repoel)
 618
 619     else:
 620         repoel.setAttribute("name", config['repo_name'])
 621         if config['repo_maxage'] != 0:
 622             repoel.setAttribute("maxage", str(config['repo_maxage']))
 623         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 624         repoel.setAttribute("url", config['repo_url'])
 625         addElement('description', config['repo_description'], doc, repoel)
 626
 627     repoel.setAttribute("version", "12")
 628     repoel.setAttribute("timestamp", str(int(time.time())))
 629
 630     if 'repo_keyalias' in config:
 631
 632         # Generate a certificate fingerprint the same way keytool does it
 633         # (but with slightly different formatting)
 634         def cert_fingerprint(data):
 635             digest = hashlib.sha256(data).digest()
 636             ret = []
 637             ret.append(' '.join("%02X" % ord(b) for b in digest))
 638             return " ".join(ret)
 639
 640         def extract_pubkey():
 641             p = FDroidPopen(['keytool', '-exportcert',
 642                                   '-alias', config['repo_keyalias'],
 643                                   '-keystore', config['keystore'],
 644                                   '-storepass:file', config['keystorepassfile']]
 645                             + config['smartcardoptions'])
 646             if p.returncode != 0:
 647                 msg = "Failed to get repo pubkey!"
 648                 if config['keystore'] == 'NONE':
 649                     msg += ' Is your crypto smartcard plugged in?'
 650                 logging.critical(msg)
 651                 sys.exit(1)
 652             global repo_pubkey_fingerprint
 653             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 654             return "".join("%02x" % ord(b) for b in p.stdout)
 655
 656         repoel.setAttribute("pubkey", extract_pubkey())
 657
 658     root.appendChild(repoel)
 659
 660     for app in apps:
 661
 662         if app['Disabled'] is not None:
 663             continue
 664
 665         # Get a list of the apks for this app...
 666         apklist = []
 667         for apk in apks:
 668             if apk['id'] == app['id']:
 669                 apklist.append(apk)
 670
 671         if len(apklist) == 0:
 672             continue
 673
 674         apel = doc.createElement("application")
 675         apel.setAttribute("id", app['id'])
 676         root.appendChild(apel)
 677
 678         addElement('id', app['id'], doc, apel)
 679         if 'added' in app:
 680             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 681         if 'lastupdated' in app:
 682             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 683         addElement('name', app['Name'], doc, apel)
 684         addElement('summary', app['Summary'], doc, apel)
 685         if app['icon']:
 686             addElement('icon', app['icon'], doc, apel)
 687         def linkres(link):
 688             for app in apps:
 689                 if app['id'] == link:
 690                     return ("fdroid.app:" + link, app['Name'])
 691             raise MetaDataException("Cannot resolve app id " + link)
 692         addElement('desc',
 693                 metadata.description_html(app['Description'], linkres), doc, apel)
 694         addElement('license', app['License'], doc, apel)
 695         if 'Categories' in app:
 696             addElement('categories', ','.join(app["Categories"]), doc, apel)
 697             # We put the first (primary) category in LAST, which will have
 698             # the desired effect of making clients that only understand one
 699             # category see that one.
 700             addElement('category', app["Categories"][0], doc, apel)
 701         addElement('web', app['Web Site'], doc, apel)
 702         addElement('source', app['Source Code'], doc, apel)
 703         addElement('tracker', app['Issue Tracker'], doc, apel)
 704         if app['Donate']:
 705             addElement('donate', app['Donate'], doc, apel)
 706         if app['Bitcoin']:
 707             addElement('bitcoin', app['Bitcoin'], doc, apel)
 708         if app['Litecoin']:
 709             addElement('litecoin', app['Litecoin'], doc, apel)
 710         if app['Dogecoin']:
 711             addElement('dogecoin', app['Dogecoin'], doc, apel)
 712         if app['FlattrID']:
 713             addElement('flattr', app['FlattrID'], doc, apel)
 714
 715         # These elements actually refer to the current version (i.e. which
 716         # one is recommended. They are historically mis-named, and need
 717         # changing, but stay like this for now to support existing clients.
 718         addElement('marketversion', app['Current Version'], doc, apel)
 719         addElement('marketvercode', app['Current Version Code'], doc, apel)
 720
 721         if app['AntiFeatures']:
 722             af = app['AntiFeatures'].split(',')
 723             # TODO: Temporarily not including UpstreamNonFree in the index,
 724             # because current F-Droid clients do not understand it, and also
 725             # look ugly when they encounter an unknown antifeature. This
 726             # filtering can be removed in time...
 727             if 'UpstreamNonFree' in af:
 728                 af.remove('UpstreamNonFree')
 729             if af:
 730                 addElement('antifeatures', ','.join(af), doc, apel)
 731         if app['Provides']:
 732             pv = app['Provides'].split(',')
 733             addElement('provides', ','.join(pv), doc, apel)
 734         if app['Requires Root']:
 735             addElement('requirements', 'root', doc, apel)
 736
 737         # Sort the apk list into version order, just so the web site
 738         # doesn't have to do any work by default...
 739         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 740
 741         # Check for duplicates - they will make the client unhappy...
 742         for i in range(len(apklist) - 1):
 743             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 744                 logging.critical("duplicate versions: '%s' - '%s'" % (
 745                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 746                 sys.exit(1)
 747
 748         for apk in apklist:
 749             apkel = doc.createElement("package")
 750             apel.appendChild(apkel)
 751             addElement('version', apk['version'], doc, apkel)
 752             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 753             addElement('apkname', apk['apkname'], doc, apkel)
 754             if 'srcname' in apk:
 755                 addElement('srcname', apk['srcname'], doc, apkel)
 756             for hash_type in ['sha256']:
 757                 if not hash_type in apk:
 758                     continue
 759                 hashel = doc.createElement("hash")
 760                 hashel.setAttribute("type", hash_type)
 761                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 762                 apkel.appendChild(hashel)
 763             addElement('sig', apk['sig'], doc, apkel)
 764             addElement('size', str(apk['size']), doc, apkel)
 765             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 766             if 'maxsdkversion' in apk:
 767                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 768             if 'added' in apk:
 769                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 770             if app['Requires Root']:
 771                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 772                     apk['permissions'].append('ACCESS_SUPERUSER')
 773
 774             if len(apk['permissions']) > 0:
 775                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 776             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 777                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 778             if len(apk['features']) > 0:
 779                 addElement('features', ','.join(apk['features']), doc, apkel)
 780
 781     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 782     if options.pretty:
 783         output = doc.toprettyxml()
 784     else:
 785         output = doc.toxml()
 786     of.write(output)
 787     of.close()
 788
 789     if 'repo_keyalias' in config:
 790
 791         logging.info("Creating signed index with this key:")
 792         logging.info("SHA256: %s" % repo_pubkey_fingerprint)
 793
 794         #Create a jar of the index...
 795         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 796         if p.returncode != 0:
 797             logging.critical("Failed to create jar file")
 798             sys.exit(1)
 799
 800         # Sign the index...
 801         args = ['jarsigner', '-keystore', config['keystore'],
 802                 '-storepass:file', config['keystorepassfile'],
 803                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 804                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 805         if config['keystore'] == 'NONE':
 806             args += config['smartcardoptions']
 807         else:  # smardcards never use -keypass
 808             args += ['-keypass:file', config['keypassfile']]
 809         p = FDroidPopen(args)
 810         # TODO keypass should be sent via stdin
 811         if p.returncode != 0:
 812             logging.info("Failed to sign index")
 813             sys.exit(1)
 814
 815     # Copy the repo icon into the repo directory...
 816     icon_dir = os.path.join(repodir, 'icons')
 817     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 818     shutil.copyfile(config['repo_icon'], iconfilename)
 819
 820     # Write a category list in the repo to allow quick access...
 821     catdata = ''
 822     for cat in categories:
 823         catdata += cat + '\n'
 824     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 825     f.write(catdata)
 826     f.close()
 827
 828
 829
 830 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 831
 832     for app in apps:
 833
 834         # Get a list of the apks for this app...
 835         apklist = []
 836         for apk in apks:
 837             if apk['id'] == app['id']:
 838                 apklist.append(apk)
 839
 840         # Sort the apk list into version order...
 841         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 842
 843         if app['Archive Policy']:
 844             keepversions = int(app['Archive Policy'][:-9])
 845         else:
 846             keepversions = defaultkeepversions
 847
 848         if len(apklist) > keepversions:
 849             for apk in apklist[keepversions:]:
 850                 logging.info("Moving " + apk['apkname'] + " to archive")
 851                 shutil.move(os.path.join(repodir, apk['apkname']),
 852                     os.path.join(archivedir, apk['apkname']))
 853                 if 'srcname' in apk:
 854                     shutil.move(os.path.join(repodir, apk['srcname']),
 855                         os.path.join(archivedir, apk['srcname']))
 856                 archapks.append(apk)
 857                 apks.remove(apk)
 858
 859
 860 config = None
 861 options = None
 862
 863 def main():
 864
 865     global config, options
 866
 867     # Parse command line...
 868     parser = OptionParser()
 869     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 870                       help="Create skeleton metadata files that are missing")
 871     parser.add_option("-v", "--verbose", action="store_true", default=False,
 872                       help="Spew out even more information than normal")
 873     parser.add_option("-q", "--quiet", action="store_true", default=False,
 874                       help="Restrict output to warnings and errors")
 875     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 876                       help="Report on build data status")
 877     parser.add_option("-i", "--interactive", default=False, action="store_true",
 878                       help="Interactively ask about things that need updating.")
 879     parser.add_option("-I", "--icons", action="store_true", default=False,
 880                       help="Resize all the icons exceeding the max pixel size and exit")
 881     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 882                       help="Specify editor to use in interactive mode. Default "+
 883                           "is /etc/alternatives/editor")
 884     parser.add_option("-w", "--wiki", default=False, action="store_true",
 885                       help="Update the wiki")
 886     parser.add_option("", "--pretty", action="store_true", default=False,
 887                       help="Produce human-readable index.xml")
 888     parser.add_option("--clean", action="store_true", default=False,
 889                       help="Clean update - don't uses caches, reprocess all apks")
 890     (options, args) = parser.parse_args()
 891
 892     config = common.read_config(options)
 893
 894     repodirs = ['repo']
 895     if config['archive_older'] != 0:
 896         repodirs.append('archive')
 897         if not os.path.exists('archive'):
 898             os.mkdir('archive')
 899
 900     if options.icons:
 901         resize_all_icons(repodirs)
 902         sys.exit(0)
 903
 904     # Get all apps...
 905     apps = metadata.read_metadata()
 906
 907     # Generate a list of categories...
 908     categories = set()
 909     for app in apps:
 910         categories.update(app['Categories'])
 911
 912     # Read known apks data (will be updated and written back when we've finished)
 913     knownapks = common.KnownApks()
 914
 915     # Gather information about all the apk files in the repo directory, using
 916     # cached data if possible.
 917     apkcachefile = os.path.join('tmp', 'apkcache')
 918     if not options.clean and os.path.exists(apkcachefile):
 919         with open(apkcachefile, 'rb') as cf:
 920             apkcache = pickle.load(cf)
 921     else:
 922         apkcache = {}
 923     cachechanged = False
 924
 925     delete_disabled_builds(apps, apkcache, repodirs)
 926
 927     # Scan all apks in the main repo
 928     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 929     if cc:
 930         cachechanged = True
 931
 932     # Scan the archive repo for apks as well
 933     if len(repodirs) > 1:
 934         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 935         if cc:
 936             cachechanged = True
 937     else:
 938         archapks = []
 939
 940     # Some information from the apks needs to be applied up to the application
 941     # level. When doing this, we use the info from the most recent version's apk.
 942     # We deal with figuring out when the app was added and last updated at the
 943     # same time.
 944     for app in apps:
 945         bestver = 0
 946         added = None
 947         lastupdated = None
 948         for apk in apks + archapks:
 949             if apk['id'] == app['id']:
 950                 if apk['versioncode'] > bestver:
 951                     bestver = apk['versioncode']
 952                     bestapk = apk
 953
 954                 if 'added' in apk:
 955                     if not added or apk['added'] < added:
 956                         added = apk['added']
 957                     if not lastupdated or apk['added'] > lastupdated:
 958                         lastupdated = apk['added']
 959
 960         if added:
 961             app['added'] = added
 962         else:
 963             logging.warn("Don't know when " + app['id'] + " was added")
 964         if lastupdated:
 965             app['lastupdated'] = lastupdated
 966         else:
 967             logging.warn("Don't know when " + app['id'] + " was last updated")
 968
 969         if bestver == 0:
 970             if app['Name'] is None:
 971                 app['Name'] = app['id']
 972             app['icon'] = None
 973             logging.warn("Application " + app['id'] + " has no packages")
 974         else:
 975             if app['Name'] is None:
 976                 app['Name'] = bestapk['name']
 977             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 978
 979     # Sort the app list by name, then the web site doesn't have to by default.
 980     # (we had to wait until we'd scanned the apks to do this, because mostly the
 981     # name comes from there!)
 982     apps = sorted(apps, key=lambda app: app['Name'].upper())
 983
 984     # Generate warnings for apk's with no metadata (or create skeleton
 985     # metadata files, if requested on the command line)
 986     for apk in apks:
 987         found = False
 988         for app in apps:
 989             if app['id'] == apk['id']:
 990                 found = True
 991                 break
 992         if not found:
 993             if options.createmeta:
 994                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 995                 f.write("License:Unknown\n")
 996                 f.write("Web Site:\n")
 997                 f.write("Source Code:\n")
 998                 f.write("Issue Tracker:\n")
 999                 f.write("Summary:" + apk['name'] + "\n")
1000                 f.write("Description:\n")
1001                 f.write(apk['name'] + "\n")
1002                 f.write(".\n")
1003                 f.close()
1004                 logging.info("Generated skeleton metadata for " + apk['id'])
1005             else:
1006                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata - removing")
1007                 rmf = os.path.join(repodirs[0], apk['apkname'])
1008                 if not os.path.exists(rmf):
1009                     logging.error("Could not find {0} to remove it".format(rmf))
1010                 else:
1011                     os.remove(rmf)
1012
1013     if len(repodirs) > 1:
1014         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1015
1016     # Make the index for the main repo...
1017     make_index(apps, apks, repodirs[0], False, categories)
1018
1019     # If there's an archive repo,  make the index for it. We already scanned it
1020     # earlier on.
1021     if len(repodirs) > 1:
1022         make_index(apps, archapks, repodirs[1], True, categories)
1023
1024     if config['update_stats']:
1025
1026         # Update known apks info...
1027         knownapks.writeifchanged()
1028
1029         # Generate latest apps data for widget
1030         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1031             data = ''
1032             for line in file(os.path.join('stats', 'latestapps.txt')):
1033                 appid = line.rstrip()
1034                 data += appid + "\t"
1035                 for app in apps:
1036                     if app['id'] == appid:
1037                         data += app['Name'] + "\t"
1038                         if app['icon'] is not None:
1039                             data += app['icon'] + "\t"
1040                         data += app['License'] + "\n"
1041                         break
1042             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1043             f.write(data)
1044             f.close()
1045
1046     if cachechanged:
1047         with open(apkcachefile, 'wb') as cf:
1048             pickle.dump(apkcache, cf)
1049
1050     # Update the wiki...
1051     if options.wiki:
1052         update_wiki(apps, apks + archapks)
1053
1054     logging.info("Finished.")
1055
1056 if __name__ == "__main__":
1057     main()
1058