fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37 def update_wiki(apps, apks):
  38     """Update the wiki
  39
  40     :param apps: fully populated list of all applications
  41     :param apks: all apks, except...
  42     """
  43     print "Updating wiki"
  44     wikicat = 'Apps'
  45     wikiredircat = 'App Redirects'
  46     import mwclient
  47     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  48             path=config['wiki_path'])
  49     site.login(config['wiki_user'], config['wiki_password'])
  50     generated_pages = {}
  51     generated_redirects = {}
  52     for app in apps:
  53         wikidata = ''
  54         if app['Disabled']:
  55             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  56         if app['AntiFeatures']:
  57             for af in app['AntiFeatures'].split(','):
  58                 wikidata += '{{AntiFeature|' + af + '}}\n'
  59         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n'%(
  60                 app['id'],
  61                 app['Name'],
  62                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  63                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  64                 app['Source Code'],
  65                 app['Issue Tracker'],
  66                 app['Web Site'],
  67                 app['Donate'],
  68                 app['FlattrID'],
  69                 app['Bitcoin'],
  70                 app['Litecoin'],
  71                 app['License'],
  72                 app.get('Requires Root', 'No'))
  73
  74         wikidata += app['Summary']
  75         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  76
  77         wikidata += "=Description=\n"
  78         wikidata += metadata.description_wiki(app['Description']) + "\n"
  79
  80         wikidata += "=Maintainer Notes=\n"
  81         if 'Maintainer Notes' in app:
  82             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
  83         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
  84
  85         # Get a list of all packages for this application...
  86         apklist = []
  87         gotcurrentver = False
  88         cantupdate = False
  89         buildfails = False
  90         for apk in apks:
  91             if apk['id'] == app['id']:
  92                 if str(apk['versioncode']) == app['Current Version Code']:
  93                     gotcurrentver = True
  94                 apklist.append(apk)
  95         # Include ones we can't build, as a special case...
  96         for thisbuild in app['builds']:
  97             if 'disable' in thisbuild:
  98                 if thisbuild['vercode'] == app['Current Version Code']:
  99                     cantupdate = True
 100                 apklist.append({
 101                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 102                         'versioncode': int(thisbuild['vercode']),
 103                         'version': thisbuild['version'],
 104                         'buildproblem': thisbuild['disable']
 105                     })
 106             else:
 107                 builtit = False
 108                 for apk in apklist:
 109                     if apk['versioncode'] == int(thisbuild['vercode']):
 110                         builtit = True
 111                         break
 112                 if not builtit:
 113                     buildfails = True
 114                     apklist.append({
 115                             'versioncode': int(thisbuild['vercode']),
 116                             'version': thisbuild['version'],
 117                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 118                         })
 119         # Sort with most recent first...
 120         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 121
 122         wikidata += "=Versions=\n"
 123         if len(apklist) == 0:
 124             wikidata += "We currently have no versions of this app available."
 125         elif not gotcurrentver:
 126             wikidata += "We don't have the current version of this app."
 127         else:
 128             wikidata += "We have the current version of this app."
 129         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 130         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 131         if len(app['No Source Since']) > 0:
 132             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 133         if len(app['Current Version']) > 0:
 134             wikidata += "The current (recommended) version is " + app['Current Version']
 135             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 136         validapks = 0
 137         for apk in apklist:
 138             wikidata += "==" + apk['version'] + "==\n"
 139
 140             if 'buildproblem' in apk:
 141                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 142             else:
 143                 validapks += 1
 144                 wikidata += "This version is built and signed by "
 145                 if 'srcname' in apk:
 146                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 147                 else:
 148                     wikidata += "the original developer.\n\n"
 149             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 150
 151         wikidata += '\n[[Category:' + wikicat + ']]\n'
 152         if len(app['No Source Since']) > 0:
 153             wikidata += '\n[[Category:Apps missing source code]]\n'
 154         elif validapks == 0 and not app['Disabled']:
 155             wikidata += '\n[[Category:Apps with no packages]]\n'
 156         elif cantupdate and not app['Disabled']:
 157             wikidata += "\n[[Category:Apps we can't update]]\n"
 158         elif buildfails and not app['Disabled']:
 159             wikidata += "\n[[Category:Apps with failing builds]]\n"
 160         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 161             wikidata += '\n[[Category:Apps to Update]]\n'
 162         if app['Disabled']:
 163             wikidata += '\n[[Category:Apps that are disabled]]\n'
 164         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 165             wikidata += '\n[[Category:Apps with no update check]]\n'
 166
 167         # We can't have underscores in the page name, even if they're in
 168         # the package ID, because MediaWiki messes with them...
 169         pagename = app['id'].replace('_', ' ')
 170
 171         # Drop a trailing newline, because mediawiki is going to drop it anyway
 172         # and it we don't we'll think the page has changed when it hasn't...
 173         if wikidata.endswith('\n'):
 174             wikidata = wikidata[:-1]
 175
 176         generated_pages[pagename] = wikidata
 177
 178         # Make a redirect from the name to the ID too, unless there's
 179         # already an existing page with the name and it isn't a redirect.
 180         noclobber = False
 181         apppagename = app['Name'].replace('_', ' ')
 182         apppagename = apppagename.replace('{', '')
 183         apppagename = apppagename.replace('}', ' ')
 184         apppagename = apppagename.replace(':', ' ')
 185         # Drop double spaces caused mostly by replacing ':' above
 186         apppagename = apppagename.replace('  ', ' ')
 187         for expagename in site.allpages(prefix=apppagename,
 188                 filterredir='nonredirects', generator=False):
 189             if expagename == apppagename:
 190                 noclobber = True
 191         # Another reason not to make the redirect page is if the app name
 192         # is the same as it's ID, because that will overwrite the real page
 193         # with an redirect to itself! (Although it seems like an odd
 194         # scenario this happens a lot, e.g. where there is metadata but no
 195         # builds or binaries to extract a name from.
 196         if apppagename == pagename:
 197             noclobber = True
 198         if not noclobber:
 199             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 200
 201     for tcat, genp in [(wikicat, generated_pages),
 202             (wikiredircat, generated_redirects)]:
 203         catpages = site.Pages['Category:' + tcat]
 204         existingpages = []
 205         for page in catpages:
 206             existingpages.append(page.name)
 207             if page.name in genp:
 208                 pagetxt = page.edit()
 209                 if pagetxt != genp[page.name]:
 210                     print "Updating modified page " + page.name
 211                     page.save(genp[page.name], summary='Auto-updated')
 212                 else:
 213                     if options.verbose:
 214                         print "Page " + page.name + " is unchanged"
 215             else:
 216                 print "Deleting page " + page.name
 217                 page.delete('No longer published')
 218         for pagename, text in genp.items():
 219             if options.verbose:
 220                 print "Checking " + pagename
 221             if not pagename in existingpages:
 222                 print "Creating page " + pagename
 223                 try:
 224                     newpage = site.Pages[pagename]
 225                     newpage.save(text, summary='Auto-created')
 226                 except:
 227                     print "...FAILED to create page"
 228
 229     # Purge server cache to ensure counts are up to date
 230     site.pages['Repository Maintenance'].purge()
 231
 232 def delete_disabled_builds(apps, apkcache, repodirs):
 233     """Delete disabled build outputs.
 234
 235     :param apps: list of all applications, as per metadata.read_metadata
 236     :param apkcache: current apk cache information
 237     :param repodirs: the repo directories to process
 238     """
 239     for app in apps:
 240         for build in app['builds']:
 241             if 'disable' in build:
 242                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 243                 for repodir in repodirs:
 244                     apkpath = os.path.join(repodir, apkfilename)
 245                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 246                     for name in [apkpath, srcpath]:
 247                         if os.path.exists(name):
 248                             print "Deleting disabled build output " + apkfilename
 249                             os.remove(name)
 250                 if apkfilename in apkcache:
 251                     del apkcache[apkfilename]
 252
 253 def resize_icon(iconpath):
 254     try:
 255         im = Image.open(iconpath)
 256         if any(length > config['max_icon_size'] for length in im.size):
 257             print iconpath, "is too large:", im.size
 258             im.thumbnail((config['max_icon_size'], config['max_icon_size']),
 259                     Image.ANTIALIAS)
 260             print iconpath, "new size:", im.size
 261             im.save(iconpath, "PNG")
 262         else:
 263             if options.verbose:
 264                 print iconpath, "is small enough:", im.size
 265     except Exception,e:
 266         print "ERROR: Failed processing {0} - {1}".format(iconpath, e)
 267
 268 def resize_all_icons(repodirs):
 269     """Resize all icons that exceed the max size
 270
 271     :param apps: list of all applications, as per metadata.read_metadata
 272     :param repodirs: the repo directories to process
 273     """
 274     for repodir in repodirs:
 275         for iconpath in glob.glob(os.path.join(repodir, 'icons', '*.png')):
 276             resize_icon(iconpath)
 277
 278 def scan_apks(apps, apkcache, repodir, knownapks):
 279     """Scan the apks in the given repo directory.
 280
 281     This also extracts the icons.
 282
 283     :param apps: list of all applications, as per metadata.read_metadata
 284     :param apkcache: current apk cache information
 285     :param repodir: repo directory to scan
 286     :param knownapks: known apks info
 287     :returns: (apks, cachechanged) where apks is a list of apk information,
 288               and cachechanged is True if the apkcache got changed.
 289     """
 290
 291     cachechanged = False
 292
 293     icon_dir = os.path.join(repodir ,'icons')
 294     # Delete and re-create the icon directory...
 295     if options.clean and os.path.exists(icon_dir):
 296         shutil.rmtree(icon_dir)
 297     if not os.path.exists(icon_dir):
 298         os.makedirs(icon_dir)
 299     apks = []
 300     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 301     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 302     vername_pat = re.compile(".*versionName='([^']*)'.*")
 303     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 304     icon_pat = re.compile(".*icon='([^']+?)'.*")
 305     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 306     string_pat = re.compile(".*'([^']*)'.*")
 307     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 308
 309         apkfilename = apkfile[len(repodir) + 1:]
 310         if ' ' in apkfilename:
 311             print "No spaces in APK filenames!"
 312             sys.exit(1)
 313
 314         if apkfilename in apkcache:
 315             if options.verbose:
 316                 print "Reading " + apkfilename + " from cache"
 317             thisinfo = apkcache[apkfilename]
 318
 319         else:
 320
 321             if not options.quiet:
 322                 print "Processing " + apkfilename
 323             thisinfo = {}
 324             thisinfo['apkname'] = apkfilename
 325             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 326             if os.path.exists(os.path.join(repodir, srcfilename)):
 327                 thisinfo['srcname'] = srcfilename
 328             thisinfo['size'] = os.path.getsize(apkfile)
 329             thisinfo['permissions'] = []
 330             thisinfo['features'] = []
 331             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 332                                   'dump', 'badging', apkfile],
 333                                  stdout=subprocess.PIPE)
 334             output = p.communicate()[0]
 335             if options.verbose:
 336                 print output
 337             if p.returncode != 0:
 338                 print "ERROR: Failed to get apk information"
 339                 sys.exit(1)
 340             for line in output.splitlines():
 341                 if line.startswith("package:"):
 342                     try:
 343                         thisinfo['id'] = re.match(name_pat, line).group(1)
 344                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 345                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 346                     except Exception, e:
 347                         print "Package matching failed: " + str(e)
 348                         print "Line was: " + line
 349                         sys.exit(1)
 350                 elif line.startswith("application:"):
 351                     thisinfo['name'] = re.match(label_pat, line).group(1)
 352                     match = re.match(icon_pat, line)
 353                     if match:
 354                         thisinfo['iconsrc'] = match.group(1)
 355                 elif line.startswith("sdkVersion:"):
 356                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 357                 elif line.startswith("native-code:"):
 358                     thisinfo['nativecode'] = []
 359                     for arch in line[13:].split(' '):
 360                         thisinfo['nativecode'].append(arch[1:-1])
 361                 elif line.startswith("uses-permission:"):
 362                     perm = re.match(string_pat, line).group(1)
 363                     if perm.startswith("android.permission."):
 364                         perm = perm[19:]
 365                     thisinfo['permissions'].append(perm)
 366                 elif line.startswith("uses-feature:"):
 367                     perm = re.match(string_pat, line).group(1)
 368                     #Filter out this, it's only added with the latest SDK tools and
 369                     #causes problems for lots of apps.
 370                     if (perm != "android.hardware.screen.portrait" and
 371                         perm != "android.hardware.screen.landscape"):
 372                         if perm.startswith("android.feature."):
 373                             perm = perm[16:]
 374                         thisinfo['features'].append(perm)
 375
 376             if not 'sdkversion' in thisinfo:
 377                 print "  WARNING: no SDK version information found"
 378                 thisinfo['sdkversion'] = 0
 379
 380             # Check for debuggable apks...
 381             if common.isApkDebuggable(apkfile, config):
 382                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 383
 384             # Calculate the sha256...
 385             sha = hashlib.sha256()
 386             with open(apkfile, 'rb') as f:
 387                 while True:
 388                     t = f.read(1024)
 389                     if len(t) == 0:
 390                         break
 391                     sha.update(t)
 392                 thisinfo['sha256'] = sha.hexdigest()
 393
 394             # Get the signature (or md5 of, to be precise)...
 395             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 396             if not os.path.exists(getsig_dir + "/getsig.class"):
 397                 print "ERROR: getsig.class not found. To fix:"
 398                 print "\tcd " + getsig_dir
 399                 print "\t./make.sh"
 400                 sys.exit(1)
 401             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 402                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 403             output = p.communicate()[0]
 404             if options.verbose:
 405                 print output
 406             if p.returncode != 0 or not output.startswith('Result:'):
 407                 print "ERROR: Failed to get apk signature"
 408                 sys.exit(1)
 409             thisinfo['sig'] = output[7:].strip()
 410
 411             # Extract the icon file...
 412             if 'iconsrc' in thisinfo:
 413                 apk = zipfile.ZipFile(apkfile, 'r')
 414                 thisinfo['icon'] = (thisinfo['id'] + '.' +
 415                     str(thisinfo['versioncode']) + '.png')
 416                 iconpath = os.path.join(icon_dir, thisinfo['icon'])
 417                 try:
 418                     iconfile = open(iconpath, 'wb')
 419                     iconfile.write(apk.read(thisinfo['iconsrc']))
 420                     iconfile.close()
 421                 except:
 422                     print "WARNING: Error retrieving icon file"
 423                 apk.close()
 424
 425                 resize_icon(iconpath)
 426
 427             # Record in known apks, getting the added date at the same time..
 428             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 429             if added:
 430                 thisinfo['added'] = added
 431
 432             apkcache[apkfilename] = thisinfo
 433             cachechanged = True
 434
 435         apks.append(thisinfo)
 436
 437     return apks, cachechanged
 438
 439
 440 repo_pubkey_fingerprint = None
 441
 442 def make_index(apps, apks, repodir, archive, categories):
 443     """Make a repo index.
 444
 445     :param apps: fully populated apps list
 446     :param apks: full populated apks list
 447     :param repodir: the repo directory
 448     :param archive: True if this is the archive repo, False if it's the
 449                     main one.
 450     :param categories: list of categories
 451     """
 452
 453     doc = Document()
 454
 455     def addElement(name, value, doc, parent):
 456         el = doc.createElement(name)
 457         el.appendChild(doc.createTextNode(value))
 458         parent.appendChild(el)
 459     def addElementCDATA(name, value, doc, parent):
 460         el = doc.createElement(name)
 461         el.appendChild(doc.createCDATASection(value))
 462         parent.appendChild(el)
 463
 464     root = doc.createElement("fdroid")
 465     doc.appendChild(root)
 466
 467     repoel = doc.createElement("repo")
 468     if archive:
 469         repoel.setAttribute("name", config['archive_name'])
 470         if config['repo_maxage'] != 0:
 471             repoel.setAttribute("maxage", str(config['repo_maxage']))
 472         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 473         repoel.setAttribute("url", config['archive_url'])
 474         addElement('description', config['archive_description'], doc, repoel)
 475     else:
 476         repoel.setAttribute("name", config['repo_name'])
 477         if config['repo_maxage'] != 0:
 478             repoel.setAttribute("maxage", str(config['repo_maxage']))
 479         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 480         repoel.setAttribute("url", config['repo_url'])
 481         addElement('description', config['repo_description'], doc, repoel)
 482     repoel.setAttribute("timestamp", str(int(time.time())))
 483
 484     if config['repo_keyalias'] is not None:
 485
 486         # Generate a certificate fingerprint the same way keytool does it
 487         # (but with slightly different formatting)
 488         def cert_fingerprint(data):
 489             digest = hashlib.sha1(data).digest()
 490             ret = []
 491             for i in range(4):
 492                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 493             return " ".join(ret)
 494
 495         def extract_pubkey():
 496             p = subprocess.Popen(['keytool', '-exportcert',
 497                                   '-alias', config['repo_keyalias'],
 498                                   '-keystore', config['keystore'],
 499                                   '-storepass', config['keystorepass']],
 500                                  stdout=subprocess.PIPE)
 501             cert = p.communicate()[0]
 502             if p.returncode != 0:
 503                 print "ERROR: Failed to get repo pubkey"
 504                 sys.exit(1)
 505             global repo_pubkey_fingerprint
 506             repo_pubkey_fingerprint = cert_fingerprint(cert)
 507             return "".join("%02x" % ord(b) for b in cert)
 508
 509         repoel.setAttribute("pubkey", extract_pubkey())
 510
 511     root.appendChild(repoel)
 512
 513     for app in apps:
 514
 515         if app['Disabled'] is not None:
 516             continue
 517
 518         # Get a list of the apks for this app...
 519         apklist = []
 520         for apk in apks:
 521             if apk['id'] == app['id']:
 522                 apklist.append(apk)
 523
 524         if len(apklist) == 0:
 525             continue
 526
 527         apel = doc.createElement("application")
 528         apel.setAttribute("id", app['id'])
 529         root.appendChild(apel)
 530
 531         addElement('id', app['id'], doc, apel)
 532         if 'added' in app:
 533             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 534         if 'lastupdated' in app:
 535             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 536         addElement('name', app['Name'], doc, apel)
 537         addElement('summary', app['Summary'], doc, apel)
 538         if app['icon'] is not None:
 539             addElement('icon', app['icon'], doc, apel)
 540         def linkres(link):
 541             for app in apps:
 542                 if app['id'] == link:
 543                     return ("fdroid.app:" + link, app['Name'])
 544             raise MetaDataException("Cannot resolve app id " + link)
 545         addElement('desc',
 546                 metadata.description_html(app['Description'], linkres), doc, apel)
 547         addElement('license', app['License'], doc, apel)
 548         if 'Categories' in app:
 549             appcategories = [c.strip() for c in app['Categories'].split(',')]
 550             addElement('categories', ','.join(appcategories), doc, apel)
 551             # We put the first (primary) category in LAST, which will have
 552             # the desired effect of making clients that only understand one
 553             # category see that one.
 554             addElement('category', appcategories[0], doc, apel)
 555         addElement('web', app['Web Site'], doc, apel)
 556         addElement('source', app['Source Code'], doc, apel)
 557         addElement('tracker', app['Issue Tracker'], doc, apel)
 558         if app['Donate'] is not None:
 559             addElement('donate', app['Donate'], doc, apel)
 560         if app['Bitcoin'] is not None:
 561             addElement('bitcoin', app['Bitcoin'], doc, apel)
 562         if app['Litecoin'] is not None:
 563             addElement('litecoin', app['Litecoin'], doc, apel)
 564         if app['FlattrID'] is not None:
 565             addElement('flattr', app['FlattrID'], doc, apel)
 566
 567         # These elements actually refer to the current version (i.e. which
 568         # one is recommended. They are historically mis-named, and need
 569         # changing, but stay like this for now to support existing clients.
 570         addElement('marketversion', app['Current Version'], doc, apel)
 571         addElement('marketvercode', app['Current Version Code'], doc, apel)
 572
 573         if app['AntiFeatures']:
 574             af = app['AntiFeatures'].split(',')
 575             # TODO: Temporarily not including UpstreamNonFree in the index,
 576             # because current F-Droid clients do not understand it, and also
 577             # look ugly when they encounter an unknown antifeature. This
 578             # filtering can be removed in time...
 579             if 'UpstreamNonFree' in af:
 580                 af.remove('UpstreamNonFree')
 581             if af:
 582                 addElement('antifeatures', ','.join(af), doc, apel)
 583         if app['Requires Root']:
 584             addElement('requirements', 'root', doc, apel)
 585
 586         # Sort the apk list into version order, just so the web site
 587         # doesn't have to do any work by default...
 588         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 589
 590         # Check for duplicates - they will make the client unhappy...
 591         for i in range(len(apklist) - 1):
 592             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 593                 print "ERROR - duplicate versions"
 594                 print apklist[i]['apkname']
 595                 print apklist[i+1]['apkname']
 596                 sys.exit(1)
 597
 598         for apk in apklist:
 599             apkel = doc.createElement("package")
 600             apel.appendChild(apkel)
 601             addElement('version', apk['version'], doc, apkel)
 602             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 603             addElement('apkname', apk['apkname'], doc, apkel)
 604             if 'srcname' in apk:
 605                 addElement('srcname', apk['srcname'], doc, apkel)
 606             for hash_type in ['sha256']:
 607                 if not hash_type in apk:
 608                     continue
 609                 hashel = doc.createElement("hash")
 610                 hashel.setAttribute("type", hash_type)
 611                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 612                 apkel.appendChild(hashel)
 613             addElement('sig', apk['sig'], doc, apkel)
 614             addElement('size', str(apk['size']), doc, apkel)
 615             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 616             if 'added' in apk:
 617                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 618             if app['Requires Root']:
 619                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 620                     apk['permissions'].append('ACCESS_SUPERUSER')
 621
 622             if len(apk['permissions']) > 0:
 623                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 624             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 625                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 626             if len(apk['features']) > 0:
 627                 addElement('features', ','.join(apk['features']), doc, apkel)
 628
 629     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 630     if options.pretty:
 631         output = doc.toprettyxml()
 632     else:
 633         output = doc.toxml()
 634     of.write(output)
 635     of.close()
 636
 637     if config['repo_keyalias'] is not None:
 638
 639         if not options.quiet:
 640             print "Creating signed index."
 641             print "Key fingerprint:", repo_pubkey_fingerprint
 642
 643         #Create a jar of the index...
 644         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 645             cwd=repodir, stdout=subprocess.PIPE)
 646         output = p.communicate()[0]
 647         if options.verbose:
 648             print output
 649         if p.returncode != 0:
 650             print "ERROR: Failed to create jar file"
 651             sys.exit(1)
 652
 653         # Sign the index...
 654         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 655             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 656             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 657             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 658         output = p.communicate()[0]
 659         if p.returncode != 0:
 660             print "Failed to sign index"
 661             print output
 662             sys.exit(1)
 663         if options.verbose:
 664             print output
 665
 666     # Copy the repo icon into the repo directory...
 667     icon_dir=os.path.join(repodir ,'icons')
 668     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 669     shutil.copyfile(config['repo_icon'], iconfilename)
 670
 671     # Write a category list in the repo to allow quick access...
 672     catdata = ''
 673     for cat in categories:
 674         catdata += cat + '\n'
 675     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 676     f.write(catdata)
 677     f.close()
 678
 679
 680
 681 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 682
 683     for app in apps:
 684
 685         # Get a list of the apks for this app...
 686         apklist = []
 687         for apk in apks:
 688             if apk['id'] == app['id']:
 689                 apklist.append(apk)
 690
 691         # Sort the apk list into version order...
 692         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 693
 694         if app['Archive Policy']:
 695             keepversions = int(app['Archive Policy'][:-9])
 696         else:
 697             keepversions = defaultkeepversions
 698
 699         if len(apklist) > keepversions:
 700             for apk in apklist[keepversions:]:
 701                 print "Moving " + apk['apkname'] + " to archive"
 702                 shutil.move(os.path.join(repodir, apk['apkname']),
 703                     os.path.join(archivedir, apk['apkname']))
 704                 if 'srcname' in apk:
 705                     shutil.move(os.path.join(repodir, apk['srcname']),
 706                         os.path.join(archivedir, apk['srcname']))
 707                 apks.remove(apk)
 708
 709
 710 config = None
 711 options = None
 712
 713 def main():
 714
 715     global config, options
 716
 717     # Parse command line...
 718     parser = OptionParser()
 719     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 720                       help="Create skeleton metadata files that are missing")
 721     parser.add_option("-v", "--verbose", action="store_true", default=False,
 722                       help="Spew out even more information than normal")
 723     parser.add_option("-q", "--quiet", action="store_true", default=False,
 724                       help="No output, except for warnings and errors")
 725     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 726                       help="Report on build data status")
 727     parser.add_option("-i", "--interactive", default=False, action="store_true",
 728                       help="Interactively ask about things that need updating.")
 729     parser.add_option("-I", "--icons", action="store_true", default=False,
 730                       help="Resize all the icons exceeding the max pixel size and exit")
 731     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 732                       help="Specify editor to use in interactive mode. Default "+
 733                           "is /etc/alternatives/editor")
 734     parser.add_option("-w", "--wiki", default=False, action="store_true",
 735                       help="Update the wiki")
 736     parser.add_option("", "--pretty", action="store_true", default=False,
 737                       help="Produce human-readable index.xml")
 738     parser.add_option("--clean", action="store_true", default=False,
 739                       help="Clean update - don't uses caches, reprocess all apks")
 740     (options, args) = parser.parse_args()
 741
 742     config = common.read_config(options)
 743
 744     repodirs = ['repo']
 745     if config['archive_older'] != 0:
 746         repodirs.append('archive')
 747         if not os.path.exists('archive'):
 748             os.mkdir('archive')
 749
 750     if options.icons:
 751         resize_all_icons(repodirs)
 752         sys.exit(0)
 753
 754     # Get all apps...
 755     apps = metadata.read_metadata()
 756
 757     # Generate a list of categories...
 758     categories = []
 759     for app in apps:
 760         cats = app['Categories'].split(',')
 761         for cat in cats:
 762             if cat not in categories:
 763                 categories.append(cat)
 764
 765     # Read known apks data (will be updated and written back when we've finished)
 766     knownapks = common.KnownApks()
 767
 768     # Gather information about all the apk files in the repo directory, using
 769     # cached data if possible.
 770     apkcachefile = os.path.join('tmp', 'apkcache')
 771     if not options.clean and os.path.exists(apkcachefile):
 772         with open(apkcachefile, 'rb') as cf:
 773             apkcache = pickle.load(cf)
 774     else:
 775         apkcache = {}
 776     cachechanged = False
 777
 778     delete_disabled_builds(apps, apkcache, repodirs)
 779
 780     # Scan all apks in the main repo
 781     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 782     if cc:
 783         cachechanged = True
 784
 785     # Scan the archive repo for apks as well
 786     if len(repodirs) > 1:
 787         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 788         if cc:
 789             cachechanged = True
 790     else:
 791         archapks = []
 792
 793     # Some information from the apks needs to be applied up to the application
 794     # level. When doing this, we use the info from the most recent version's apk.
 795     # We deal with figuring out when the app was added and last updated at the
 796     # same time.
 797     for app in apps:
 798         bestver = 0
 799         added = None
 800         lastupdated = None
 801         for apk in apks + archapks:
 802             if apk['id'] == app['id']:
 803                 if apk['versioncode'] > bestver:
 804                     bestver = apk['versioncode']
 805                     bestapk = apk
 806
 807                 if 'added' in apk:
 808                     if not added or apk['added'] < added:
 809                         added = apk['added']
 810                     if not lastupdated or apk['added'] > lastupdated:
 811                         lastupdated = apk['added']
 812
 813         if added:
 814             app['added'] = added
 815         else:
 816             if options.verbose:
 817                 print "WARNING: Don't know when " + app['id'] + " was added"
 818         if lastupdated:
 819             app['lastupdated'] = lastupdated
 820         else:
 821             if options.verbose:
 822                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 823
 824         if bestver == 0:
 825             if app['Name'] is None:
 826                 app['Name'] = app['id']
 827             app['icon'] = None
 828             if options.verbose and app['Disabled'] is None:
 829                 print "WARNING: Application " + app['id'] + " has no packages"
 830         else:
 831             if app['Name'] is None:
 832                 app['Name'] = bestapk['name']
 833             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 834
 835     # Sort the app list by name, then the web site doesn't have to by default.
 836     # (we had to wait until we'd scanned the apks to do this, because mostly the
 837     # name comes from there!)
 838     apps = sorted(apps, key=lambda app: app['Name'].upper())
 839
 840     # Generate warnings for apk's with no metadata (or create skeleton
 841     # metadata files, if requested on the command line)
 842     for apk in apks:
 843         found = False
 844         for app in apps:
 845             if app['id'] == apk['id']:
 846                 found = True
 847                 break
 848         if not found:
 849             if options.createmeta:
 850                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 851                 f.write("License:Unknown\n")
 852                 f.write("Web Site:\n")
 853                 f.write("Source Code:\n")
 854                 f.write("Issue Tracker:\n")
 855                 f.write("Summary:" + apk['name'] + "\n")
 856                 f.write("Description:\n")
 857                 f.write(apk['name'] + "\n")
 858                 f.write(".\n")
 859                 f.close()
 860                 print "Generated skeleton metadata for " + apk['id']
 861             else:
 862                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 863                 print "       " + apk['name'] + " - " + apk['version']
 864
 865     if len(repodirs) > 1:
 866         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 867
 868     # Make the index for the main repo...
 869     make_index(apps, apks, repodirs[0], False, categories)
 870
 871     # If there's an archive repo,  make the index for it. We already scanned it
 872     # earlier on.
 873     if len(repodirs) > 1:
 874         make_index(apps, archapks, repodirs[1], True, categories)
 875
 876     if config['update_stats']:
 877
 878         # Update known apks info...
 879         knownapks.writeifchanged()
 880
 881         # Generate latest apps data for widget
 882         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 883             data = ''
 884             for line in file(os.path.join('stats', 'latestapps.txt')):
 885                 appid = line.rstrip()
 886                 data += appid + "\t"
 887                 for app in apps:
 888                     if app['id'] == appid:
 889                         data += app['Name'] + "\t"
 890                         if app['icon'] is not None:
 891                             data += app['icon'] + "\t"
 892                         data += app['License'] + "\n"
 893                         break
 894             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
 895             f.write(data)
 896             f.close()
 897
 898     if cachechanged:
 899         with open(apkcachefile, 'wb') as cf:
 900             pickle.dump(apkcache, cf)
 901
 902     # Update the wiki...
 903     if options.wiki:
 904         update_wiki(apps, apks + archapks)
 905
 906     print "Finished."
 907
 908 if __name__ == "__main__":
 909     main()
 910