fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def dpi_to_px(density):
  42     return (int(density) * 48) / 160
  43
  44 def px_to_dpi(px):
  45     return (int(px) * 160) / 48
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         if validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         if cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         if buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 191             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 192
 193         # We can't have underscores in the page name, even if they're in
 194         # the package ID, because MediaWiki messes with them...
 195         pagename = app['id'].replace('_', ' ')
 196
 197         # Drop a trailing newline, because mediawiki is going to drop it anyway
 198         # and it we don't we'll think the page has changed when it hasn't...
 199         if wikidata.endswith('\n'):
 200             wikidata = wikidata[:-1]
 201
 202         generated_pages[pagename] = wikidata
 203
 204         # Make a redirect from the name to the ID too, unless there's
 205         # already an existing page with the name and it isn't a redirect.
 206         noclobber = False
 207         apppagename = app['Name'].replace('_', ' ')
 208         apppagename = apppagename.replace('{', '')
 209         apppagename = apppagename.replace('}', ' ')
 210         apppagename = apppagename.replace(':', ' ')
 211         # Drop double spaces caused mostly by replacing ':' above
 212         apppagename = apppagename.replace('  ', ' ')
 213         for expagename in site.allpages(prefix=apppagename,
 214                 filterredir='nonredirects', generator=False):
 215             if expagename == apppagename:
 216                 noclobber = True
 217         # Another reason not to make the redirect page is if the app name
 218         # is the same as it's ID, because that will overwrite the real page
 219         # with an redirect to itself! (Although it seems like an odd
 220         # scenario this happens a lot, e.g. where there is metadata but no
 221         # builds or binaries to extract a name from.
 222         if apppagename == pagename:
 223             noclobber = True
 224         if not noclobber:
 225             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 226
 227     for tcat, genp in [(wikicat, generated_pages),
 228             (wikiredircat, generated_redirects)]:
 229         catpages = site.Pages['Category:' + tcat]
 230         existingpages = []
 231         for page in catpages:
 232             existingpages.append(page.name)
 233             if page.name in genp:
 234                 pagetxt = page.edit()
 235                 if pagetxt != genp[page.name]:
 236                     print "Updating modified page " + page.name
 237                     page.save(genp[page.name], summary='Auto-updated')
 238                 else:
 239                     if options.verbose:
 240                         print "Page " + page.name + " is unchanged"
 241             else:
 242                 print "Deleting page " + page.name
 243                 page.delete('No longer published')
 244         for pagename, text in genp.items():
 245             if options.verbose:
 246                 print "Checking " + pagename
 247             if not pagename in existingpages:
 248                 print "Creating page " + pagename
 249                 try:
 250                     newpage = site.Pages[pagename]
 251                     newpage.save(text, summary='Auto-created')
 252                 except:
 253                     print "...FAILED to create page"
 254
 255     # Purge server cache to ensure counts are up to date
 256     site.pages['Repository Maintenance'].purge()
 257
 258 def delete_disabled_builds(apps, apkcache, repodirs):
 259     """Delete disabled build outputs.
 260
 261     :param apps: list of all applications, as per metadata.read_metadata
 262     :param apkcache: current apk cache information
 263     :param repodirs: the repo directories to process
 264     """
 265     for app in apps:
 266         for build in app['builds']:
 267             if 'disable' in build:
 268                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 269                 for repodir in repodirs:
 270                     apkpath = os.path.join(repodir, apkfilename)
 271                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 272                     for name in [apkpath, srcpath]:
 273                         if os.path.exists(name):
 274                             print "Deleting disabled build output " + apkfilename
 275                             os.remove(name)
 276                 if apkfilename in apkcache:
 277                     del apkcache[apkfilename]
 278
 279 def resize_icon(iconpath, density):
 280
 281     if not os.path.isfile(iconpath):
 282         return
 283
 284     try:
 285         im = Image.open(iconpath)
 286         size = dpi_to_px(density)
 287
 288         if any(length > size for length in im.size):
 289             oldsize = im.size
 290             im.thumbnail((size, size), Image.ANTIALIAS)
 291             print iconpath, "was too large at", oldsize, "- new size is", im.size
 292             im.save(iconpath, "PNG")
 293
 294         else:
 295             if options.verbose:
 296                 print iconpath, "is small enough:", im.size
 297
 298     except Exception,e:
 299         print "WARNING: Failed resizing {0} - {1}".format(iconpath, e)
 300
 301 def resize_all_icons(repodirs):
 302     """Resize all icons that exceed the max size
 303
 304     :param repodirs: the repo directories to process
 305     """
 306     for repodir in repodirs:
 307         for density in get_densities():
 308             icon_dir = get_icon_dir(repodir, density)
 309             icon_glob = os.path.join(icon_dir, '*.png')
 310             for iconpath in glob.glob(icon_glob):
 311                 resize_icon(iconpath, density)
 312
 313 def scan_apks(apps, apkcache, repodir, knownapks):
 314     """Scan the apks in the given repo directory.
 315
 316     This also extracts the icons.
 317
 318     :param apps: list of all applications, as per metadata.read_metadata
 319     :param apkcache: current apk cache information
 320     :param repodir: repo directory to scan
 321     :param knownapks: known apks info
 322     :returns: (apks, cachechanged) where apks is a list of apk information,
 323               and cachechanged is True if the apkcache got changed.
 324     """
 325
 326     cachechanged = False
 327
 328     icon_dirs = get_icon_dirs(repodir)
 329     for icon_dir in icon_dirs:
 330         if os.path.exists(icon_dir):
 331             if options.clean:
 332                 shutil.rmtree(icon_dir)
 333                 os.makedirs(icon_dir)
 334         else:
 335             os.makedirs(icon_dir)
 336
 337     apks = []
 338     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 339     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 340     vername_pat = re.compile(".*versionName='([^']*)'.*")
 341     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 342     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 343     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 344     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 345     string_pat = re.compile(".*'([^']*)'.*")
 346     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 347
 348         apkfilename = apkfile[len(repodir) + 1:]
 349         if ' ' in apkfilename:
 350             print "No spaces in APK filenames!"
 351             sys.exit(1)
 352
 353         if apkfilename in apkcache:
 354             if options.verbose:
 355                 print "Reading " + apkfilename + " from cache"
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             if not options.quiet:
 361                 print "Processing " + apkfilename
 362             thisinfo = {}
 363             thisinfo['apkname'] = apkfilename
 364             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 365             if os.path.exists(os.path.join(repodir, srcfilename)):
 366                 thisinfo['srcname'] = srcfilename
 367             thisinfo['size'] = os.path.getsize(apkfile)
 368             thisinfo['permissions'] = []
 369             thisinfo['features'] = []
 370             thisinfo['icons_src'] = {}
 371             thisinfo['icons'] = {}
 372             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 373                                   'dump', 'badging', apkfile],
 374                                  stdout=subprocess.PIPE)
 375             output = p.communicate()[0]
 376             if p.returncode != 0:
 377                 print "ERROR: Failed to get apk information"
 378                 sys.exit(1)
 379             for line in output.splitlines():
 380                 if line.startswith("package:"):
 381                     try:
 382                         thisinfo['id'] = re.match(name_pat, line).group(1)
 383                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 384                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 385                     except Exception, e:
 386                         print "Package matching failed: " + str(e)
 387                         print "Line was: " + line
 388                         sys.exit(1)
 389                 elif line.startswith("application:"):
 390                     thisinfo['name'] = re.match(label_pat, line).group(1)
 391                     # Keep path to non-dpi icon in case we need it
 392                     match = re.match(icon_pat_nodpi, line)
 393                     if match:
 394                         thisinfo['icons_src']['-1'] = match.group(1)
 395                 elif line.startswith("launchable-activity:"):
 396                     # Only use launchable-activity as fallback to application
 397                     if '-1' not in thisinfo['icons_src']:
 398                         match = re.match(icon_pat_nodpi, line)
 399                         if match:
 400                             thisinfo['icons_src']['-1'] = match.group(1)
 401                 elif line.startswith("application-icon-"):
 402                     match = re.match(icon_pat, line)
 403                     if match:
 404                         density = match.group(1)
 405                         path = match.group(2)
 406                         thisinfo['icons_src'][density] = path
 407                 elif line.startswith("sdkVersion:"):
 408                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 print "  WARNING: no SDK version information found"
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 print "ERROR: getsig.class not found. To fix:"
 450                 print "\tcd " + getsig_dir
 451                 print "\t./make.sh"
 452                 sys.exit(1)
 453             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 454                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 455             output = p.communicate()[0]
 456             if options.verbose:
 457                 print output
 458             if p.returncode != 0 or not output.startswith('Result:'):
 459                 print "ERROR: Failed to get apk signature"
 460                 sys.exit(1)
 461             thisinfo['sig'] = output[7:].strip()
 462
 463             apk = zipfile.ZipFile(apkfile, 'r')
 464
 465             iconfilename = "%s.%s.png" % (
 466                     thisinfo['id'],
 467                     thisinfo['versioncode'])
 468
 469             # Extract the icon file...
 470             densities = get_densities()
 471             empty_densities = []
 472             for density in densities:
 473                 if density not in thisinfo['icons_src']:
 474                     empty_densities.append(density)
 475                     continue
 476                 iconsrc = thisinfo['icons_src'][density]
 477                 icon_dir = get_icon_dir(repodir, density)
 478                 icondest = os.path.join(icon_dir, iconfilename)
 479
 480                 try:
 481                     iconfile = open(icondest, 'wb')
 482                     iconfile.write(apk.read(iconsrc))
 483                     iconfile.close()
 484                     thisinfo['icons'][density] = iconfilename
 485
 486                 except:
 487                     print "WARNING: Error retrieving icon file"
 488                     del thisinfo['icons'][density]
 489                     del thisinfo['icons_src'][density]
 490                     empty_densities.append(density)
 491
 492             if '-1' in thisinfo['icons_src']:
 493                 iconsrc = thisinfo['icons_src']['-1']
 494                 iconpath = os.path.join(
 495                         get_icon_dir(repodir, None), iconfilename)
 496                 iconfile = open(iconpath, 'wb')
 497                 iconfile.write(apk.read(iconsrc))
 498                 iconfile.close()
 499                 try:
 500                     im = Image.open(iconpath)
 501                     dpi = px_to_dpi(im.size[0])
 502                     for density in densities:
 503                         if density in thisinfo['icons']:
 504                             break
 505                         if density == densities[-1] or dpi >= int(density):
 506                             thisinfo['icons'][density] = iconfilename
 507                             shutil.move(iconpath,
 508                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 509                             empty_densities.remove(density)
 510                             break
 511                 except Exception,e:
 512                     print "WARNING: Failed reading {0} - {1}".format(iconpath, e)
 513
 514             if thisinfo['icons']:
 515                 thisinfo['icon'] = iconfilename
 516
 517             apk.close()
 518
 519             # First try resizing down to not lose quality
 520             last_density = None
 521             for density in densities:
 522                 if density not in empty_densities:
 523                     last_density = density
 524                     continue
 525                 if last_density is None:
 526                     continue
 527                 if options.verbose:
 528                     print "Density %s not available, resizing down from %s" % (
 529                             density, last_density)
 530
 531                 last_iconpath = os.path.join(
 532                         get_icon_dir(repodir, last_density), iconfilename)
 533                 iconpath = os.path.join(
 534                         get_icon_dir(repodir, density), iconfilename)
 535                 im = Image.open(last_iconpath)
 536                 size = dpi_to_px(density)
 537
 538                 im.thumbnail((size, size), Image.ANTIALIAS)
 539                 im.save(iconpath, "PNG")
 540                 empty_densities.remove(density)
 541
 542             # Then just copy from the highest resolution available
 543             last_density = None
 544             for density in reversed(densities):
 545                 if density not in empty_densities:
 546                     last_density = density
 547                     continue
 548                 if last_density is None:
 549                     continue
 550                 if options.verbose:
 551                     print "Density %s not available, copying from lower density %s" % (
 552                             density, last_density)
 553
 554                 shutil.copyfile(
 555                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 556                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 557
 558                 empty_densities.remove(density)
 559
 560             for density in densities:
 561                 icon_dir = get_icon_dir(repodir, density)
 562                 icondest = os.path.join(icon_dir, iconfilename)
 563                 resize_icon(icondest, density)
 564
 565             # Copy from icons-mdpi to icons since mdpi is the baseline density
 566             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 567             if os.path.isfile(baseline):
 568                 shutil.copyfile(baseline,
 569                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 570
 571             # Record in known apks, getting the added date at the same time..
 572             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 573             if added:
 574                 thisinfo['added'] = added
 575
 576             apkcache[apkfilename] = thisinfo
 577             cachechanged = True
 578
 579         apks.append(thisinfo)
 580
 581     return apks, cachechanged
 582
 583
 584 repo_pubkey_fingerprint = None
 585
 586 def make_index(apps, apks, repodir, archive, categories):
 587     """Make a repo index.
 588
 589     :param apps: fully populated apps list
 590     :param apks: full populated apks list
 591     :param repodir: the repo directory
 592     :param archive: True if this is the archive repo, False if it's the
 593                     main one.
 594     :param categories: list of categories
 595     """
 596
 597     doc = Document()
 598
 599     def addElement(name, value, doc, parent):
 600         el = doc.createElement(name)
 601         el.appendChild(doc.createTextNode(value))
 602         parent.appendChild(el)
 603     def addElementCDATA(name, value, doc, parent):
 604         el = doc.createElement(name)
 605         el.appendChild(doc.createCDATASection(value))
 606         parent.appendChild(el)
 607
 608     root = doc.createElement("fdroid")
 609     doc.appendChild(root)
 610
 611     repoel = doc.createElement("repo")
 612
 613     if archive:
 614         repoel.setAttribute("name", config['archive_name'])
 615         if config['repo_maxage'] != 0:
 616             repoel.setAttribute("maxage", str(config['repo_maxage']))
 617         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 618         repoel.setAttribute("url", config['archive_url'])
 619         addElement('description', config['archive_description'], doc, repoel)
 620
 621     else:
 622         repoel.setAttribute("name", config['repo_name'])
 623         if config['repo_maxage'] != 0:
 624             repoel.setAttribute("maxage", str(config['repo_maxage']))
 625         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 626         repoel.setAttribute("url", config['repo_url'])
 627         addElement('description', config['repo_description'], doc, repoel)
 628
 629     repoel.setAttribute("version", "11")
 630     repoel.setAttribute("timestamp", str(int(time.time())))
 631
 632     if config['repo_keyalias']:
 633
 634         # Generate a certificate fingerprint the same way keytool does it
 635         # (but with slightly different formatting)
 636         def cert_fingerprint(data):
 637             digest = hashlib.sha1(data).digest()
 638             ret = []
 639             for i in range(4):
 640                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 641             return " ".join(ret)
 642
 643         def extract_pubkey():
 644             p = subprocess.Popen(['keytool', '-exportcert',
 645                                   '-alias', config['repo_keyalias'],
 646                                   '-keystore', config['keystore'],
 647                                   '-storepass', config['keystorepass']],
 648                                  stdout=subprocess.PIPE)
 649             cert = p.communicate()[0]
 650             if p.returncode != 0:
 651                 print "ERROR: Failed to get repo pubkey"
 652                 sys.exit(1)
 653             global repo_pubkey_fingerprint
 654             repo_pubkey_fingerprint = cert_fingerprint(cert)
 655             return "".join("%02x" % ord(b) for b in cert)
 656
 657         repoel.setAttribute("pubkey", extract_pubkey())
 658
 659     root.appendChild(repoel)
 660
 661     for app in apps:
 662
 663         if app['Disabled'] is not None:
 664             continue
 665
 666         # Get a list of the apks for this app...
 667         apklist = []
 668         for apk in apks:
 669             if apk['id'] == app['id']:
 670                 apklist.append(apk)
 671
 672         if len(apklist) == 0:
 673             continue
 674
 675         apel = doc.createElement("application")
 676         apel.setAttribute("id", app['id'])
 677         root.appendChild(apel)
 678
 679         addElement('id', app['id'], doc, apel)
 680         if 'added' in app:
 681             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 682         if 'lastupdated' in app:
 683             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 684         addElement('name', app['Name'], doc, apel)
 685         addElement('summary', app['Summary'], doc, apel)
 686         if app['icon']:
 687             addElement('icon', app['icon'], doc, apel)
 688         def linkres(link):
 689             for app in apps:
 690                 if app['id'] == link:
 691                     return ("fdroid.app:" + link, app['Name'])
 692             raise MetaDataException("Cannot resolve app id " + link)
 693         addElement('desc',
 694                 metadata.description_html(app['Description'], linkres), doc, apel)
 695         addElement('license', app['License'], doc, apel)
 696         if 'Categories' in app:
 697             appcategories = [c.strip() for c in app['Categories'].split(',')]
 698             addElement('categories', ','.join(appcategories), doc, apel)
 699             # We put the first (primary) category in LAST, which will have
 700             # the desired effect of making clients that only understand one
 701             # category see that one.
 702             addElement('category', appcategories[0], doc, apel)
 703         addElement('web', app['Web Site'], doc, apel)
 704         addElement('source', app['Source Code'], doc, apel)
 705         addElement('tracker', app['Issue Tracker'], doc, apel)
 706         if app['Donate']:
 707             addElement('donate', app['Donate'], doc, apel)
 708         if app['Bitcoin']:
 709             addElement('bitcoin', app['Bitcoin'], doc, apel)
 710         if app['Litecoin']:
 711             addElement('litecoin', app['Litecoin'], doc, apel)
 712         if app['Dogecoin']:
 713             addElement('dogecoin', app['Dogecoin'], doc, apel)
 714         if app['FlattrID']:
 715             addElement('flattr', app['FlattrID'], doc, apel)
 716
 717         # These elements actually refer to the current version (i.e. which
 718         # one is recommended. They are historically mis-named, and need
 719         # changing, but stay like this for now to support existing clients.
 720         addElement('marketversion', app['Current Version'], doc, apel)
 721         addElement('marketvercode', app['Current Version Code'], doc, apel)
 722
 723         if app['AntiFeatures']:
 724             af = app['AntiFeatures'].split(',')
 725             # TODO: Temporarily not including UpstreamNonFree in the index,
 726             # because current F-Droid clients do not understand it, and also
 727             # look ugly when they encounter an unknown antifeature. This
 728             # filtering can be removed in time...
 729             if 'UpstreamNonFree' in af:
 730                 af.remove('UpstreamNonFree')
 731             if af:
 732                 addElement('antifeatures', ','.join(af), doc, apel)
 733         if app['Provides']:
 734             pv = app['Provides'].split(',')
 735             addElement('provides', ','.join(pv), doc, apel)
 736         if app['Requires Root']:
 737             addElement('requirements', 'root', doc, apel)
 738
 739         # Sort the apk list into version order, just so the web site
 740         # doesn't have to do any work by default...
 741         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 742
 743         # Check for duplicates - they will make the client unhappy...
 744         for i in range(len(apklist) - 1):
 745             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 746                 print "ERROR - duplicate versions"
 747                 print apklist[i]['apkname']
 748                 print apklist[i+1]['apkname']
 749                 sys.exit(1)
 750
 751         for apk in apklist:
 752             apkel = doc.createElement("package")
 753             apel.appendChild(apkel)
 754             addElement('version', apk['version'], doc, apkel)
 755             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 756             addElement('apkname', apk['apkname'], doc, apkel)
 757             if 'srcname' in apk:
 758                 addElement('srcname', apk['srcname'], doc, apkel)
 759             for hash_type in ['sha256']:
 760                 if not hash_type in apk:
 761                     continue
 762                 hashel = doc.createElement("hash")
 763                 hashel.setAttribute("type", hash_type)
 764                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 765                 apkel.appendChild(hashel)
 766             addElement('sig', apk['sig'], doc, apkel)
 767             addElement('size', str(apk['size']), doc, apkel)
 768             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 769             if 'added' in apk:
 770                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 771             if app['Requires Root']:
 772                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 773                     apk['permissions'].append('ACCESS_SUPERUSER')
 774
 775             if len(apk['permissions']) > 0:
 776                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 777             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 778                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 779             if len(apk['features']) > 0:
 780                 addElement('features', ','.join(apk['features']), doc, apkel)
 781
 782     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 783     if options.pretty:
 784         output = doc.toprettyxml()
 785     else:
 786         output = doc.toxml()
 787     of.write(output)
 788     of.close()
 789
 790     if config['repo_keyalias'] is not None:
 791
 792         if not options.quiet:
 793             print "Creating signed index."
 794             print "Key fingerprint:", repo_pubkey_fingerprint
 795
 796         #Create a jar of the index...
 797         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 798             cwd=repodir, stdout=subprocess.PIPE)
 799         output = p.communicate()[0]
 800         if options.verbose:
 801             print output
 802         if p.returncode != 0:
 803             print "ERROR: Failed to create jar file"
 804             sys.exit(1)
 805
 806         # Sign the index...
 807         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 808             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 809             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 810             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 811         output = p.communicate()[0]
 812         if p.returncode != 0:
 813             print "Failed to sign index"
 814             print output
 815             sys.exit(1)
 816         if options.verbose:
 817             print output
 818
 819     # Copy the repo icon into the repo directory...
 820     icon_dir = os.path.join(repodir ,'icons')
 821     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 822     shutil.copyfile(config['repo_icon'], iconfilename)
 823
 824     # Write a category list in the repo to allow quick access...
 825     catdata = ''
 826     for cat in categories:
 827         catdata += cat + '\n'
 828     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 829     f.write(catdata)
 830     f.close()
 831
 832
 833
 834 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 835
 836     for app in apps:
 837
 838         # Get a list of the apks for this app...
 839         apklist = []
 840         for apk in apks:
 841             if apk['id'] == app['id']:
 842                 apklist.append(apk)
 843
 844         # Sort the apk list into version order...
 845         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 846
 847         if app['Archive Policy']:
 848             keepversions = int(app['Archive Policy'][:-9])
 849         else:
 850             keepversions = defaultkeepversions
 851
 852         if len(apklist) > keepversions:
 853             for apk in apklist[keepversions:]:
 854                 print "Moving " + apk['apkname'] + " to archive"
 855                 shutil.move(os.path.join(repodir, apk['apkname']),
 856                     os.path.join(archivedir, apk['apkname']))
 857                 if 'srcname' in apk:
 858                     shutil.move(os.path.join(repodir, apk['srcname']),
 859                         os.path.join(archivedir, apk['srcname']))
 860                 archapks.append(apk)
 861                 apks.remove(apk)
 862
 863
 864 config = None
 865 options = None
 866
 867 def main():
 868
 869     global config, options
 870
 871     # Parse command line...
 872     parser = OptionParser()
 873     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 874                       help="Create skeleton metadata files that are missing")
 875     parser.add_option("-v", "--verbose", action="store_true", default=False,
 876                       help="Spew out even more information than normal")
 877     parser.add_option("-q", "--quiet", action="store_true", default=False,
 878                       help="No output, except for warnings and errors")
 879     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 880                       help="Report on build data status")
 881     parser.add_option("-i", "--interactive", default=False, action="store_true",
 882                       help="Interactively ask about things that need updating.")
 883     parser.add_option("-I", "--icons", action="store_true", default=False,
 884                       help="Resize all the icons exceeding the max pixel size and exit")
 885     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 886                       help="Specify editor to use in interactive mode. Default "+
 887                           "is /etc/alternatives/editor")
 888     parser.add_option("-w", "--wiki", default=False, action="store_true",
 889                       help="Update the wiki")
 890     parser.add_option("", "--pretty", action="store_true", default=False,
 891                       help="Produce human-readable index.xml")
 892     parser.add_option("--clean", action="store_true", default=False,
 893                       help="Clean update - don't uses caches, reprocess all apks")
 894     (options, args) = parser.parse_args()
 895
 896     config = common.read_config(options)
 897
 898     repodirs = ['repo']
 899     if config['archive_older'] != 0:
 900         repodirs.append('archive')
 901         if not os.path.exists('archive'):
 902             os.mkdir('archive')
 903
 904     if options.icons:
 905         resize_all_icons(repodirs)
 906         sys.exit(0)
 907
 908     # Get all apps...
 909     apps = metadata.read_metadata()
 910
 911     # Generate a list of categories...
 912     categories = []
 913     for app in apps:
 914         cats = app['Categories'].split(',')
 915         for cat in cats:
 916             if cat not in categories:
 917                 categories.append(cat)
 918
 919     # Read known apks data (will be updated and written back when we've finished)
 920     knownapks = common.KnownApks()
 921
 922     # Gather information about all the apk files in the repo directory, using
 923     # cached data if possible.
 924     apkcachefile = os.path.join('tmp', 'apkcache')
 925     if not options.clean and os.path.exists(apkcachefile):
 926         with open(apkcachefile, 'rb') as cf:
 927             apkcache = pickle.load(cf)
 928     else:
 929         apkcache = {}
 930     cachechanged = False
 931
 932     delete_disabled_builds(apps, apkcache, repodirs)
 933
 934     # Scan all apks in the main repo
 935     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 936     if cc:
 937         cachechanged = True
 938
 939     # Scan the archive repo for apks as well
 940     if len(repodirs) > 1:
 941         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 942         if cc:
 943             cachechanged = True
 944     else:
 945         archapks = []
 946
 947     # Some information from the apks needs to be applied up to the application
 948     # level. When doing this, we use the info from the most recent version's apk.
 949     # We deal with figuring out when the app was added and last updated at the
 950     # same time.
 951     for app in apps:
 952         bestver = 0
 953         added = None
 954         lastupdated = None
 955         for apk in apks + archapks:
 956             if apk['id'] == app['id']:
 957                 if apk['versioncode'] > bestver:
 958                     bestver = apk['versioncode']
 959                     bestapk = apk
 960
 961                 if 'added' in apk:
 962                     if not added or apk['added'] < added:
 963                         added = apk['added']
 964                     if not lastupdated or apk['added'] > lastupdated:
 965                         lastupdated = apk['added']
 966
 967         if added:
 968             app['added'] = added
 969         else:
 970             if options.verbose:
 971                 print "WARNING: Don't know when " + app['id'] + " was added"
 972         if lastupdated:
 973             app['lastupdated'] = lastupdated
 974         else:
 975             if options.verbose:
 976                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 977
 978         if bestver == 0:
 979             if app['Name'] is None:
 980                 app['Name'] = app['id']
 981             app['icon'] = None
 982             if options.verbose and app['Disabled'] is None:
 983                 print "WARNING: Application " + app['id'] + " has no packages"
 984         else:
 985             if app['Name'] is None:
 986                 app['Name'] = bestapk['name']
 987             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 988
 989     # Sort the app list by name, then the web site doesn't have to by default.
 990     # (we had to wait until we'd scanned the apks to do this, because mostly the
 991     # name comes from there!)
 992     apps = sorted(apps, key=lambda app: app['Name'].upper())
 993
 994     # Generate warnings for apk's with no metadata (or create skeleton
 995     # metadata files, if requested on the command line)
 996     for apk in apks:
 997         found = False
 998         for app in apps:
 999             if app['id'] == apk['id']:
1000                 found = True
1001                 break
1002         if not found:
1003             if options.createmeta:
1004                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1005                 f.write("License:Unknown\n")
1006                 f.write("Web Site:\n")
1007                 f.write("Source Code:\n")
1008                 f.write("Issue Tracker:\n")
1009                 f.write("Summary:" + apk['name'] + "\n")
1010                 f.write("Description:\n")
1011                 f.write(apk['name'] + "\n")
1012                 f.write(".\n")
1013                 f.close()
1014                 print "Generated skeleton metadata for " + apk['id']
1015             else:
1016                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
1017                 print "       " + apk['name'] + " - " + apk['version']
1018
1019     if len(repodirs) > 1:
1020         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1021
1022     # Make the index for the main repo...
1023     make_index(apps, apks, repodirs[0], False, categories)
1024
1025     # If there's an archive repo,  make the index for it. We already scanned it
1026     # earlier on.
1027     if len(repodirs) > 1:
1028         make_index(apps, archapks, repodirs[1], True, categories)
1029
1030     if config['update_stats']:
1031
1032         # Update known apks info...
1033         knownapks.writeifchanged()
1034
1035         # Generate latest apps data for widget
1036         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1037             data = ''
1038             for line in file(os.path.join('stats', 'latestapps.txt')):
1039                 appid = line.rstrip()
1040                 data += appid + "\t"
1041                 for app in apps:
1042                     if app['id'] == appid:
1043                         data += app['Name'] + "\t"
1044                         if app['icon'] is not None:
1045                             data += app['icon'] + "\t"
1046                         data += app['License'] + "\n"
1047                         break
1048             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1049             f.write(data)
1050             f.close()
1051
1052     if cachechanged:
1053         with open(apkcachefile, 'wb') as cf:
1054             pickle.dump(apkcache, cf)
1055
1056     # Update the wiki...
1057     if options.wiki:
1058         update_wiki(apps, apks + archapks)
1059
1060     print "Finished."
1061
1062 if __name__ == "__main__":
1063     main()
1064