fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['480', '320', '240', '160', '120']
  40
  41 def launcher_size(density):
  42     return (int(density) * 48) / 160
  43
  44 def icon_dens_label(density):
  45     return "icon-%s" % density
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         elif validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         elif cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         elif buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190
 191         # We can't have underscores in the page name, even if they're in
 192         # the package ID, because MediaWiki messes with them...
 193         pagename = app['id'].replace('_', ' ')
 194
 195         # Drop a trailing newline, because mediawiki is going to drop it anyway
 196         # and it we don't we'll think the page has changed when it hasn't...
 197         if wikidata.endswith('\n'):
 198             wikidata = wikidata[:-1]
 199
 200         generated_pages[pagename] = wikidata
 201
 202         # Make a redirect from the name to the ID too, unless there's
 203         # already an existing page with the name and it isn't a redirect.
 204         noclobber = False
 205         apppagename = app['Name'].replace('_', ' ')
 206         apppagename = apppagename.replace('{', '')
 207         apppagename = apppagename.replace('}', ' ')
 208         apppagename = apppagename.replace(':', ' ')
 209         # Drop double spaces caused mostly by replacing ':' above
 210         apppagename = apppagename.replace('  ', ' ')
 211         for expagename in site.allpages(prefix=apppagename,
 212                 filterredir='nonredirects', generator=False):
 213             if expagename == apppagename:
 214                 noclobber = True
 215         # Another reason not to make the redirect page is if the app name
 216         # is the same as it's ID, because that will overwrite the real page
 217         # with an redirect to itself! (Although it seems like an odd
 218         # scenario this happens a lot, e.g. where there is metadata but no
 219         # builds or binaries to extract a name from.
 220         if apppagename == pagename:
 221             noclobber = True
 222         if not noclobber:
 223             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 224
 225     for tcat, genp in [(wikicat, generated_pages),
 226             (wikiredircat, generated_redirects)]:
 227         catpages = site.Pages['Category:' + tcat]
 228         existingpages = []
 229         for page in catpages:
 230             existingpages.append(page.name)
 231             if page.name in genp:
 232                 pagetxt = page.edit()
 233                 if pagetxt != genp[page.name]:
 234                     print "Updating modified page " + page.name
 235                     page.save(genp[page.name], summary='Auto-updated')
 236                 else:
 237                     if options.verbose:
 238                         print "Page " + page.name + " is unchanged"
 239             else:
 240                 print "Deleting page " + page.name
 241                 page.delete('No longer published')
 242         for pagename, text in genp.items():
 243             if options.verbose:
 244                 print "Checking " + pagename
 245             if not pagename in existingpages:
 246                 print "Creating page " + pagename
 247                 try:
 248                     newpage = site.Pages[pagename]
 249                     newpage.save(text, summary='Auto-created')
 250                 except:
 251                     print "...FAILED to create page"
 252
 253     # Purge server cache to ensure counts are up to date
 254     site.pages['Repository Maintenance'].purge()
 255
 256 def delete_disabled_builds(apps, apkcache, repodirs):
 257     """Delete disabled build outputs.
 258
 259     :param apps: list of all applications, as per metadata.read_metadata
 260     :param apkcache: current apk cache information
 261     :param repodirs: the repo directories to process
 262     """
 263     for app in apps:
 264         for build in app['builds']:
 265             if 'disable' in build:
 266                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 267                 for repodir in repodirs:
 268                     apkpath = os.path.join(repodir, apkfilename)
 269                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 270                     for name in [apkpath, srcpath]:
 271                         if os.path.exists(name):
 272                             print "Deleting disabled build output " + apkfilename
 273                             os.remove(name)
 274                 if apkfilename in apkcache:
 275                     del apkcache[apkfilename]
 276
 277 def resize_icon(iconpath, density):
 278
 279     try:
 280         im = Image.open(iconpath)
 281         size = launcher_size(density)
 282
 283         if any(length > size for length in im.size):
 284             print iconpath, "is too large:", im.size
 285             im.thumbnail((size, size), Image.ANTIALIAS)
 286             print iconpath, "new size:", im.size
 287             im.save(iconpath, "PNG")
 288
 289         else:
 290             if options.verbose:
 291                 print iconpath, "is small enough:", im.size
 292
 293     except Exception,e:
 294         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 295
 296 def resize_all_icons(repodirs):
 297     """Resize all icons that exceed the max size
 298
 299     :param repodirs: the repo directories to process
 300     """
 301     for repodir in repodirs:
 302         for density in get_densities():
 303             icon_dir = get_icon_dir(repodir, density)
 304             icon_glob = os.path.join(repodir, icon_dir, '*.png')
 305             for iconpath in glob.glob(icon_glob):
 306                 resize_icon(iconpath)
 307
 308 def scan_apks(apps, apkcache, repodir, knownapks):
 309     """Scan the apks in the given repo directory.
 310
 311     This also extracts the icons.
 312
 313     :param apps: list of all applications, as per metadata.read_metadata
 314     :param apkcache: current apk cache information
 315     :param repodir: repo directory to scan
 316     :param knownapks: known apks info
 317     :returns: (apks, cachechanged) where apks is a list of apk information,
 318               and cachechanged is True if the apkcache got changed.
 319     """
 320
 321     cachechanged = False
 322
 323     icon_dirs = get_icon_dirs(repodir)
 324     for icon_dir in icon_dirs:
 325         if os.path.exists(icon_dir):
 326             if options.clean:
 327                 shutil.rmtree(icon_dir)
 328                 os.makedirs(icon_dir)
 329         else:
 330             os.makedirs(icon_dir)
 331
 332     apks = []
 333     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 334     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 335     vername_pat = re.compile(".*versionName='([^']*)'.*")
 336     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 337     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 338     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 339     string_pat = re.compile(".*'([^']*)'.*")
 340     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 341
 342         apkfilename = apkfile[len(repodir) + 1:]
 343         if ' ' in apkfilename:
 344             print "No spaces in APK filenames!"
 345             sys.exit(1)
 346
 347         if apkfilename in apkcache:
 348             if options.verbose:
 349                 print "Reading " + apkfilename + " from cache"
 350             thisinfo = apkcache[apkfilename]
 351
 352         else:
 353
 354             if not options.quiet:
 355                 print "Processing " + apkfilename
 356             thisinfo = {}
 357             thisinfo['apkname'] = apkfilename
 358             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 359             if os.path.exists(os.path.join(repodir, srcfilename)):
 360                 thisinfo['srcname'] = srcfilename
 361             thisinfo['size'] = os.path.getsize(apkfile)
 362             thisinfo['permissions'] = []
 363             thisinfo['features'] = []
 364             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 365                                   'dump', 'badging', apkfile],
 366                                  stdout=subprocess.PIPE)
 367             output = p.communicate()[0]
 368             #if options.verbose:
 369                 #print output
 370             if p.returncode != 0:
 371                 print "ERROR: Failed to get apk information"
 372                 sys.exit(1)
 373             for line in output.splitlines():
 374                 if line.startswith("package:"):
 375                     try:
 376                         thisinfo['id'] = re.match(name_pat, line).group(1)
 377                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 378                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 379                     except Exception, e:
 380                         print "Package matching failed: " + str(e)
 381                         print "Line was: " + line
 382                         sys.exit(1)
 383                 elif line.startswith("application:"):
 384                     thisinfo['name'] = re.match(label_pat, line).group(1)
 385                 elif line.startswith("application-icon-"):
 386                     match = re.match(icon_pat, line)
 387                     if match:
 388                         density = match.group(1)
 389                         path = match.group(2)
 390                         if 'icons_src' not in thisinfo:
 391                             thisinfo['icons_src'] = {}
 392                         thisinfo['icons_src'][density] = path
 393                 elif line.startswith("sdkVersion:"):
 394                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 395                 elif line.startswith("native-code:"):
 396                     thisinfo['nativecode'] = []
 397                     for arch in line[13:].split(' '):
 398                         thisinfo['nativecode'].append(arch[1:-1])
 399                 elif line.startswith("uses-permission:"):
 400                     perm = re.match(string_pat, line).group(1)
 401                     if perm.startswith("android.permission."):
 402                         perm = perm[19:]
 403                     thisinfo['permissions'].append(perm)
 404                 elif line.startswith("uses-feature:"):
 405                     perm = re.match(string_pat, line).group(1)
 406                     #Filter out this, it's only added with the latest SDK tools and
 407                     #causes problems for lots of apps.
 408                     if (perm != "android.hardware.screen.portrait" and
 409                         perm != "android.hardware.screen.landscape"):
 410                         if perm.startswith("android.feature."):
 411                             perm = perm[16:]
 412                         thisinfo['features'].append(perm)
 413
 414             if not 'sdkversion' in thisinfo:
 415                 print "  WARNING: no SDK version information found"
 416                 thisinfo['sdkversion'] = 0
 417
 418             # Check for debuggable apks...
 419             if common.isApkDebuggable(apkfile, config):
 420                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 421
 422             # Calculate the sha256...
 423             sha = hashlib.sha256()
 424             with open(apkfile, 'rb') as f:
 425                 while True:
 426                     t = f.read(1024)
 427                     if len(t) == 0:
 428                         break
 429                     sha.update(t)
 430                 thisinfo['sha256'] = sha.hexdigest()
 431
 432             # Get the signature (or md5 of, to be precise)...
 433             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 434             if not os.path.exists(getsig_dir + "/getsig.class"):
 435                 print "ERROR: getsig.class not found. To fix:"
 436                 print "\tcd " + getsig_dir
 437                 print "\t./make.sh"
 438                 sys.exit(1)
 439             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 440                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 441             output = p.communicate()[0]
 442             if options.verbose:
 443                 print output
 444             if p.returncode != 0 or not output.startswith('Result:'):
 445                 print "ERROR: Failed to get apk signature"
 446                 sys.exit(1)
 447             thisinfo['sig'] = output[7:].strip()
 448
 449             iconfilename = "%s.%s.png" % (
 450                     thisinfo['id'],
 451                     thisinfo['versioncode'])
 452
 453             # Extract the icon file...
 454             empty_densities = []
 455             for density in get_densities():
 456                 label = icon_dens_label(density)
 457                 if density not in thisinfo['icons_src']:
 458                     empty_densities.append(density)
 459                     continue
 460                 apk = zipfile.ZipFile(apkfile, 'r')
 461                 iconsrc = thisinfo['icons_src'][density]
 462                 icon_dir = get_icon_dir(repodir, density)
 463                 icondest = os.path.join(icon_dir, iconfilename)
 464
 465                 try:
 466                     iconfile = open(icondest, 'wb')
 467                     iconfile.write(apk.read(iconsrc))
 468                     iconfile.close()
 469                     thisinfo[label] = iconfilename
 470
 471                 except:
 472                     print "WARNING: Error retrieving icon file"
 473                     del thisinfo[label]
 474                     del thisinfo['icons_src'][density]
 475                     empty_densities.append(density)
 476
 477                 apk.close()
 478
 479                 resize_icon(icondest, density)
 480
 481             # TODO: Handle empty_densities
 482
 483             # Copy from icons-mdpi to icons since mdpi is the baseline density
 484             shutil.copyfile(
 485                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 486                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 487
 488             # Record in known apks, getting the added date at the same time..
 489             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 490             if added:
 491                 thisinfo['added'] = added
 492
 493             apkcache[apkfilename] = thisinfo
 494             cachechanged = True
 495
 496         apks.append(thisinfo)
 497
 498     return apks, cachechanged
 499
 500
 501 repo_pubkey_fingerprint = None
 502
 503 def make_index(apps, apks, repodir, archive, categories):
 504     """Make a repo index.
 505
 506     :param apps: fully populated apps list
 507     :param apks: full populated apks list
 508     :param repodir: the repo directory
 509     :param archive: True if this is the archive repo, False if it's the
 510                     main one.
 511     :param categories: list of categories
 512     """
 513
 514     doc = Document()
 515
 516     def addElement(name, value, doc, parent):
 517         el = doc.createElement(name)
 518         el.appendChild(doc.createTextNode(value))
 519         parent.appendChild(el)
 520     def addElementCDATA(name, value, doc, parent):
 521         el = doc.createElement(name)
 522         el.appendChild(doc.createCDATASection(value))
 523         parent.appendChild(el)
 524
 525     root = doc.createElement("fdroid")
 526     doc.appendChild(root)
 527
 528     repoel = doc.createElement("repo")
 529
 530     if archive:
 531         repoel.setAttribute("name", config['archive_name'])
 532         if config['repo_maxage'] != 0:
 533             repoel.setAttribute("maxage", str(config['repo_maxage']))
 534         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 535         repoel.setAttribute("url", config['archive_url'])
 536         addElement('description', config['archive_description'], doc, repoel)
 537
 538     else:
 539         repoel.setAttribute("name", config['repo_name'])
 540         if config['repo_maxage'] != 0:
 541             repoel.setAttribute("maxage", str(config['repo_maxage']))
 542         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 543         repoel.setAttribute("url", config['repo_url'])
 544         addElement('description', config['repo_description'], doc, repoel)
 545
 546     repoel.setAttribute("version", "10")
 547     repoel.setAttribute("timestamp", str(int(time.time())))
 548
 549     if config['repo_keyalias']:
 550
 551         # Generate a certificate fingerprint the same way keytool does it
 552         # (but with slightly different formatting)
 553         def cert_fingerprint(data):
 554             digest = hashlib.sha1(data).digest()
 555             ret = []
 556             for i in range(4):
 557                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 558             return " ".join(ret)
 559
 560         def extract_pubkey():
 561             p = subprocess.Popen(['keytool', '-exportcert',
 562                                   '-alias', config['repo_keyalias'],
 563                                   '-keystore', config['keystore'],
 564                                   '-storepass', config['keystorepass']],
 565                                  stdout=subprocess.PIPE)
 566             cert = p.communicate()[0]
 567             if p.returncode != 0:
 568                 print "ERROR: Failed to get repo pubkey"
 569                 sys.exit(1)
 570             global repo_pubkey_fingerprint
 571             repo_pubkey_fingerprint = cert_fingerprint(cert)
 572             return "".join("%02x" % ord(b) for b in cert)
 573
 574         repoel.setAttribute("pubkey", extract_pubkey())
 575
 576     root.appendChild(repoel)
 577
 578     for app in apps:
 579
 580         if app['Disabled'] is not None:
 581             continue
 582
 583         # Get a list of the apks for this app...
 584         apklist = []
 585         for apk in apks:
 586             if apk['id'] == app['id']:
 587                 apklist.append(apk)
 588
 589         if len(apklist) == 0:
 590             continue
 591
 592         apel = doc.createElement("application")
 593         apel.setAttribute("id", app['id'])
 594         root.appendChild(apel)
 595
 596         addElement('id', app['id'], doc, apel)
 597         if 'added' in app:
 598             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 599         if 'lastupdated' in app:
 600             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 601         addElement('name', app['Name'], doc, apel)
 602         addElement('summary', app['Summary'], doc, apel)
 603         if app['icon']:
 604             addElement('icon', app['icon'], doc, apel)
 605         def linkres(link):
 606             for app in apps:
 607                 if app['id'] == link:
 608                     return ("fdroid.app:" + link, app['Name'])
 609             raise MetaDataException("Cannot resolve app id " + link)
 610         addElement('desc',
 611                 metadata.description_html(app['Description'], linkres), doc, apel)
 612         addElement('license', app['License'], doc, apel)
 613         if 'Categories' in app:
 614             appcategories = [c.strip() for c in app['Categories'].split(',')]
 615             addElement('categories', ','.join(appcategories), doc, apel)
 616             # We put the first (primary) category in LAST, which will have
 617             # the desired effect of making clients that only understand one
 618             # category see that one.
 619             addElement('category', appcategories[0], doc, apel)
 620         addElement('web', app['Web Site'], doc, apel)
 621         addElement('source', app['Source Code'], doc, apel)
 622         addElement('tracker', app['Issue Tracker'], doc, apel)
 623         if app['Donate']:
 624             addElement('donate', app['Donate'], doc, apel)
 625         if app['Bitcoin']:
 626             addElement('bitcoin', app['Bitcoin'], doc, apel)
 627         if app['Litecoin']:
 628             addElement('litecoin', app['Litecoin'], doc, apel)
 629         if app['Dogecoin']:
 630             addElement('dogecoin', app['Dogecoin'], doc, apel)
 631         if app['FlattrID']:
 632             addElement('flattr', app['FlattrID'], doc, apel)
 633
 634         # These elements actually refer to the current version (i.e. which
 635         # one is recommended. They are historically mis-named, and need
 636         # changing, but stay like this for now to support existing clients.
 637         addElement('marketversion', app['Current Version'], doc, apel)
 638         addElement('marketvercode', app['Current Version Code'], doc, apel)
 639
 640         if app['AntiFeatures']:
 641             af = app['AntiFeatures'].split(',')
 642             # TODO: Temporarily not including UpstreamNonFree in the index,
 643             # because current F-Droid clients do not understand it, and also
 644             # look ugly when they encounter an unknown antifeature. This
 645             # filtering can be removed in time...
 646             if 'UpstreamNonFree' in af:
 647                 af.remove('UpstreamNonFree')
 648             if af:
 649                 addElement('antifeatures', ','.join(af), doc, apel)
 650         if app['Provides']:
 651             pv = app['Provides'].split(',')
 652             addElement('provides', ','.join(pv), doc, apel)
 653         if app['Requires Root']:
 654             addElement('requirements', 'root', doc, apel)
 655
 656         # Sort the apk list into version order, just so the web site
 657         # doesn't have to do any work by default...
 658         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 659
 660         # Check for duplicates - they will make the client unhappy...
 661         for i in range(len(apklist) - 1):
 662             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 663                 print "ERROR - duplicate versions"
 664                 print apklist[i]['apkname']
 665                 print apklist[i+1]['apkname']
 666                 sys.exit(1)
 667
 668         for apk in apklist:
 669             apkel = doc.createElement("package")
 670             apel.appendChild(apkel)
 671             addElement('version', apk['version'], doc, apkel)
 672             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 673             addElement('apkname', apk['apkname'], doc, apkel)
 674             if 'srcname' in apk:
 675                 addElement('srcname', apk['srcname'], doc, apkel)
 676             for hash_type in ['sha256']:
 677                 if not hash_type in apk:
 678                     continue
 679                 hashel = doc.createElement("hash")
 680                 hashel.setAttribute("type", hash_type)
 681                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 682                 apkel.appendChild(hashel)
 683             addElement('sig', apk['sig'], doc, apkel)
 684             addElement('size', str(apk['size']), doc, apkel)
 685             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 686             if 'added' in apk:
 687                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 688             if app['Requires Root']:
 689                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 690                     apk['permissions'].append('ACCESS_SUPERUSER')
 691
 692             if len(apk['permissions']) > 0:
 693                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 694             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 695                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 696             if len(apk['features']) > 0:
 697                 addElement('features', ','.join(apk['features']), doc, apkel)
 698
 699     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 700     if options.pretty:
 701         output = doc.toprettyxml()
 702     else:
 703         output = doc.toxml()
 704     of.write(output)
 705     of.close()
 706
 707     if config['repo_keyalias'] is not None:
 708
 709         if not options.quiet:
 710             print "Creating signed index."
 711             print "Key fingerprint:", repo_pubkey_fingerprint
 712
 713         #Create a jar of the index...
 714         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 715             cwd=repodir, stdout=subprocess.PIPE)
 716         output = p.communicate()[0]
 717         if options.verbose:
 718             print output
 719         if p.returncode != 0:
 720             print "ERROR: Failed to create jar file"
 721             sys.exit(1)
 722
 723         # Sign the index...
 724         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 725             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 726             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 727             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 728         output = p.communicate()[0]
 729         if p.returncode != 0:
 730             print "Failed to sign index"
 731             print output
 732             sys.exit(1)
 733         if options.verbose:
 734             print output
 735
 736     # Copy the repo icon into the repo directory...
 737     icon_dir = os.path.join(repodir ,'icons')
 738     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 739     shutil.copyfile(config['repo_icon'], iconfilename)
 740
 741     # Write a category list in the repo to allow quick access...
 742     catdata = ''
 743     for cat in categories:
 744         catdata += cat + '\n'
 745     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 746     f.write(catdata)
 747     f.close()
 748
 749
 750
 751 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 752
 753     for app in apps:
 754
 755         # Get a list of the apks for this app...
 756         apklist = []
 757         for apk in apks:
 758             if apk['id'] == app['id']:
 759                 apklist.append(apk)
 760
 761         # Sort the apk list into version order...
 762         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 763
 764         if app['Archive Policy']:
 765             keepversions = int(app['Archive Policy'][:-9])
 766         else:
 767             keepversions = defaultkeepversions
 768
 769         if len(apklist) > keepversions:
 770             for apk in apklist[keepversions:]:
 771                 print "Moving " + apk['apkname'] + " to archive"
 772                 shutil.move(os.path.join(repodir, apk['apkname']),
 773                     os.path.join(archivedir, apk['apkname']))
 774                 if 'srcname' in apk:
 775                     shutil.move(os.path.join(repodir, apk['srcname']),
 776                         os.path.join(archivedir, apk['srcname']))
 777                 apks.remove(apk)
 778
 779
 780 config = None
 781 options = None
 782
 783 def main():
 784
 785     global config, options
 786
 787     # Parse command line...
 788     parser = OptionParser()
 789     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 790                       help="Create skeleton metadata files that are missing")
 791     parser.add_option("-v", "--verbose", action="store_true", default=False,
 792                       help="Spew out even more information than normal")
 793     parser.add_option("-q", "--quiet", action="store_true", default=False,
 794                       help="No output, except for warnings and errors")
 795     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 796                       help="Report on build data status")
 797     parser.add_option("-i", "--interactive", default=False, action="store_true",
 798                       help="Interactively ask about things that need updating.")
 799     parser.add_option("-I", "--icons", action="store_true", default=False,
 800                       help="Resize all the icons exceeding the max pixel size and exit")
 801     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 802                       help="Specify editor to use in interactive mode. Default "+
 803                           "is /etc/alternatives/editor")
 804     parser.add_option("-w", "--wiki", default=False, action="store_true",
 805                       help="Update the wiki")
 806     parser.add_option("", "--pretty", action="store_true", default=False,
 807                       help="Produce human-readable index.xml")
 808     parser.add_option("--clean", action="store_true", default=False,
 809                       help="Clean update - don't uses caches, reprocess all apks")
 810     (options, args) = parser.parse_args()
 811
 812     config = common.read_config(options)
 813
 814     repodirs = ['repo']
 815     if config['archive_older'] != 0:
 816         repodirs.append('archive')
 817         if not os.path.exists('archive'):
 818             os.mkdir('archive')
 819
 820     if options.icons:
 821         resize_all_icons(repodirs)
 822         sys.exit(0)
 823
 824     # Get all apps...
 825     apps = metadata.read_metadata()
 826
 827     # Generate a list of categories...
 828     categories = []
 829     for app in apps:
 830         cats = app['Categories'].split(',')
 831         for cat in cats:
 832             if cat not in categories:
 833                 categories.append(cat)
 834
 835     # Read known apks data (will be updated and written back when we've finished)
 836     knownapks = common.KnownApks()
 837
 838     # Gather information about all the apk files in the repo directory, using
 839     # cached data if possible.
 840     apkcachefile = os.path.join('tmp', 'apkcache')
 841     if not options.clean and os.path.exists(apkcachefile):
 842         with open(apkcachefile, 'rb') as cf:
 843             apkcache = pickle.load(cf)
 844     else:
 845         apkcache = {}
 846     cachechanged = False
 847
 848     delete_disabled_builds(apps, apkcache, repodirs)
 849
 850     # Scan all apks in the main repo
 851     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 852     if cc:
 853         cachechanged = True
 854
 855     # Scan the archive repo for apks as well
 856     if len(repodirs) > 1:
 857         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 858         if cc:
 859             cachechanged = True
 860     else:
 861         archapks = []
 862
 863     # Some information from the apks needs to be applied up to the application
 864     # level. When doing this, we use the info from the most recent version's apk.
 865     # We deal with figuring out when the app was added and last updated at the
 866     # same time.
 867     for app in apps:
 868         bestver = 0
 869         added = None
 870         lastupdated = None
 871         for apk in apks + archapks:
 872             if apk['id'] == app['id']:
 873                 if apk['versioncode'] > bestver:
 874                     bestver = apk['versioncode']
 875                     bestapk = apk
 876
 877                 if 'added' in apk:
 878                     if not added or apk['added'] < added:
 879                         added = apk['added']
 880                     if not lastupdated or apk['added'] > lastupdated:
 881                         lastupdated = apk['added']
 882
 883         if added:
 884             app['added'] = added
 885         #else:
 886             #if options.verbose:
 887                 #print "WARNING: Don't know when " + app['id'] + " was added"
 888         if lastupdated:
 889             app['lastupdated'] = lastupdated
 890         #else:
 891             #if options.verbose:
 892                 #print "WARNING: Don't know when " + app['id'] + " was last updated"
 893
 894         if bestver == 0:
 895             if app['Name'] is None:
 896                 app['Name'] = app['id']
 897             app['icon'] = None
 898             #if options.verbose and app['Disabled'] is None:
 899                 #print "WARNING: Application " + app['id'] + " has no packages"
 900         else:
 901             if app['Name'] is None:
 902                 app['Name'] = bestapk['name']
 903             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 904
 905     # Sort the app list by name, then the web site doesn't have to by default.
 906     # (we had to wait until we'd scanned the apks to do this, because mostly the
 907     # name comes from there!)
 908     apps = sorted(apps, key=lambda app: app['Name'].upper())
 909
 910     # Generate warnings for apk's with no metadata (or create skeleton
 911     # metadata files, if requested on the command line)
 912     for apk in apks:
 913         found = False
 914         for app in apps:
 915             if app['id'] == apk['id']:
 916                 found = True
 917                 break
 918         if not found:
 919             if options.createmeta:
 920                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 921                 f.write("License:Unknown\n")
 922                 f.write("Web Site:\n")
 923                 f.write("Source Code:\n")
 924                 f.write("Issue Tracker:\n")
 925                 f.write("Summary:" + apk['name'] + "\n")
 926                 f.write("Description:\n")
 927                 f.write(apk['name'] + "\n")
 928                 f.write(".\n")
 929                 f.close()
 930                 print "Generated skeleton metadata for " + apk['id']
 931             else:
 932                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 933                 print "       " + apk['name'] + " - " + apk['version']
 934
 935     if len(repodirs) > 1:
 936         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 937
 938     # Make the index for the main repo...
 939     make_index(apps, apks, repodirs[0], False, categories)
 940
 941     # If there's an archive repo,  make the index for it. We already scanned it
 942     # earlier on.
 943     if len(repodirs) > 1:
 944         make_index(apps, archapks, repodirs[1], True, categories)
 945
 946     if config['update_stats']:
 947
 948         # Update known apks info...
 949         knownapks.writeifchanged()
 950
 951         # Generate latest apps data for widget
 952         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 953             data = ''
 954             for line in file(os.path.join('stats', 'latestapps.txt')):
 955                 appid = line.rstrip()
 956                 data += appid + "\t"
 957                 for app in apps:
 958                     if app['id'] == appid:
 959                         data += app['Name'] + "\t"
 960                         if app['icon'] is not None:
 961                             data += app['icon'] + "\t"
 962                         data += app['License'] + "\n"
 963                         break
 964             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
 965             f.write(data)
 966             f.close()
 967
 968     if cachechanged:
 969         with open(apkcachefile, 'wb') as cf:
 970             pickle.dump(apkcache, cf)
 971
 972     # Update the wiki...
 973     if options.wiki:
 974         update_wiki(apps, apks + archapks)
 975
 976     print "Finished."
 977
 978 if __name__ == "__main__":
 979     main()
 980