fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37 def update_wiki(apps, apks):
  38     """Update the wiki
  39
  40     :param apps: fully populated list of all applications
  41     :param apks: all apks, except...
  42     """
  43     print "Updating wiki"
  44     wikicat = 'Apps'
  45     wikiredircat = 'App Redirects'
  46     import mwclient
  47     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  48             path=config['wiki_path'])
  49     site.login(config['wiki_user'], config['wiki_password'])
  50     generated_pages = {}
  51     generated_redirects = {}
  52     for app in apps:
  53         wikidata = ''
  54         if app['Disabled']:
  55             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  56         if app['AntiFeatures']:
  57             for af in app['AntiFeatures'].split(','):
  58                 wikidata += '{{AntiFeature|' + af + '}}\n'
  59         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n'%(
  60                 app['id'],
  61                 app['Name'],
  62                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  63                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  64                 app['Source Code'],
  65                 app['Issue Tracker'],
  66                 app['Web Site'],
  67                 app['Donate'],
  68                 app['FlattrID'],
  69                 app['Bitcoin'],
  70                 app['Litecoin'],
  71                 app['License'],
  72                 app.get('Requires Root', 'No'))
  73
  74         if app['Provides']:
  75             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  76
  77         wikidata += app['Summary']
  78         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  79
  80         wikidata += "=Description=\n"
  81         wikidata += metadata.description_wiki(app['Description']) + "\n"
  82
  83         wikidata += "=Maintainer Notes=\n"
  84         if 'Maintainer Notes' in app:
  85             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
  86         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
  87
  88         # Get a list of all packages for this application...
  89         apklist = []
  90         gotcurrentver = False
  91         cantupdate = False
  92         buildfails = False
  93         for apk in apks:
  94             if apk['id'] == app['id']:
  95                 if str(apk['versioncode']) == app['Current Version Code']:
  96                     gotcurrentver = True
  97                 apklist.append(apk)
  98         # Include ones we can't build, as a special case...
  99         for thisbuild in app['builds']:
 100             if 'disable' in thisbuild:
 101                 if thisbuild['vercode'] == app['Current Version Code']:
 102                     cantupdate = True
 103                 apklist.append({
 104                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 105                         'versioncode': int(thisbuild['vercode']),
 106                         'version': thisbuild['version'],
 107                         'buildproblem': thisbuild['disable']
 108                     })
 109             else:
 110                 builtit = False
 111                 for apk in apklist:
 112                     if apk['versioncode'] == int(thisbuild['vercode']):
 113                         builtit = True
 114                         break
 115                 if not builtit:
 116                     buildfails = True
 117                     apklist.append({
 118                             'versioncode': int(thisbuild['vercode']),
 119                             'version': thisbuild['version'],
 120                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 121                         })
 122         # Sort with most recent first...
 123         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 124
 125         wikidata += "=Versions=\n"
 126         if len(apklist) == 0:
 127             wikidata += "We currently have no versions of this app available."
 128         elif not gotcurrentver:
 129             wikidata += "We don't have the current version of this app."
 130         else:
 131             wikidata += "We have the current version of this app."
 132         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 133         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 134         if len(app['No Source Since']) > 0:
 135             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 136         if len(app['Current Version']) > 0:
 137             wikidata += "The current (recommended) version is " + app['Current Version']
 138             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 139         validapks = 0
 140         for apk in apklist:
 141             wikidata += "==" + apk['version'] + "==\n"
 142
 143             if 'buildproblem' in apk:
 144                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 145             else:
 146                 validapks += 1
 147                 wikidata += "This version is built and signed by "
 148                 if 'srcname' in apk:
 149                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 150                 else:
 151                     wikidata += "the original developer.\n\n"
 152             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 153
 154         wikidata += '\n[[Category:' + wikicat + ']]\n'
 155         if len(app['No Source Since']) > 0:
 156             wikidata += '\n[[Category:Apps missing source code]]\n'
 157         elif validapks == 0 and not app['Disabled']:
 158             wikidata += '\n[[Category:Apps with no packages]]\n'
 159         elif cantupdate and not app['Disabled']:
 160             wikidata += "\n[[Category:Apps we can't update]]\n"
 161         elif buildfails and not app['Disabled']:
 162             wikidata += "\n[[Category:Apps with failing builds]]\n"
 163         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 164             wikidata += '\n[[Category:Apps to Update]]\n'
 165         if app['Disabled']:
 166             wikidata += '\n[[Category:Apps that are disabled]]\n'
 167         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 168             wikidata += '\n[[Category:Apps with no update check]]\n'
 169
 170         # We can't have underscores in the page name, even if they're in
 171         # the package ID, because MediaWiki messes with them...
 172         pagename = app['id'].replace('_', ' ')
 173
 174         # Drop a trailing newline, because mediawiki is going to drop it anyway
 175         # and it we don't we'll think the page has changed when it hasn't...
 176         if wikidata.endswith('\n'):
 177             wikidata = wikidata[:-1]
 178
 179         generated_pages[pagename] = wikidata
 180
 181         # Make a redirect from the name to the ID too, unless there's
 182         # already an existing page with the name and it isn't a redirect.
 183         noclobber = False
 184         apppagename = app['Name'].replace('_', ' ')
 185         apppagename = apppagename.replace('{', '')
 186         apppagename = apppagename.replace('}', ' ')
 187         apppagename = apppagename.replace(':', ' ')
 188         # Drop double spaces caused mostly by replacing ':' above
 189         apppagename = apppagename.replace('  ', ' ')
 190         for expagename in site.allpages(prefix=apppagename,
 191                 filterredir='nonredirects', generator=False):
 192             if expagename == apppagename:
 193                 noclobber = True
 194         # Another reason not to make the redirect page is if the app name
 195         # is the same as it's ID, because that will overwrite the real page
 196         # with an redirect to itself! (Although it seems like an odd
 197         # scenario this happens a lot, e.g. where there is metadata but no
 198         # builds or binaries to extract a name from.
 199         if apppagename == pagename:
 200             noclobber = True
 201         if not noclobber:
 202             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 203
 204     for tcat, genp in [(wikicat, generated_pages),
 205             (wikiredircat, generated_redirects)]:
 206         catpages = site.Pages['Category:' + tcat]
 207         existingpages = []
 208         for page in catpages:
 209             existingpages.append(page.name)
 210             if page.name in genp:
 211                 pagetxt = page.edit()
 212                 if pagetxt != genp[page.name]:
 213                     print "Updating modified page " + page.name
 214                     page.save(genp[page.name], summary='Auto-updated')
 215                 else:
 216                     if options.verbose:
 217                         print "Page " + page.name + " is unchanged"
 218             else:
 219                 print "Deleting page " + page.name
 220                 page.delete('No longer published')
 221         for pagename, text in genp.items():
 222             if options.verbose:
 223                 print "Checking " + pagename
 224             if not pagename in existingpages:
 225                 print "Creating page " + pagename
 226                 try:
 227                     newpage = site.Pages[pagename]
 228                     newpage.save(text, summary='Auto-created')
 229                 except:
 230                     print "...FAILED to create page"
 231
 232     # Purge server cache to ensure counts are up to date
 233     site.pages['Repository Maintenance'].purge()
 234
 235 def delete_disabled_builds(apps, apkcache, repodirs):
 236     """Delete disabled build outputs.
 237
 238     :param apps: list of all applications, as per metadata.read_metadata
 239     :param apkcache: current apk cache information
 240     :param repodirs: the repo directories to process
 241     """
 242     for app in apps:
 243         for build in app['builds']:
 244             if 'disable' in build:
 245                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 246                 for repodir in repodirs:
 247                     apkpath = os.path.join(repodir, apkfilename)
 248                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 249                     for name in [apkpath, srcpath]:
 250                         if os.path.exists(name):
 251                             print "Deleting disabled build output " + apkfilename
 252                             os.remove(name)
 253                 if apkfilename in apkcache:
 254                     del apkcache[apkfilename]
 255
 256 def resize_icon(iconpath):
 257     try:
 258         im = Image.open(iconpath)
 259         if any(length > config['max_icon_size'] for length in im.size):
 260             print iconpath, "is too large:", im.size
 261             im.thumbnail((config['max_icon_size'], config['max_icon_size']),
 262                     Image.ANTIALIAS)
 263             print iconpath, "new size:", im.size
 264             im.save(iconpath, "PNG")
 265         else:
 266             if options.verbose:
 267                 print iconpath, "is small enough:", im.size
 268     except Exception,e:
 269         print "ERROR: Failed processing {0} - {1}".format(iconpath, e)
 270
 271 def resize_all_icons(repodirs):
 272     """Resize all icons that exceed the max size
 273
 274     :param apps: list of all applications, as per metadata.read_metadata
 275     :param repodirs: the repo directories to process
 276     """
 277     for repodir in repodirs:
 278         for iconpath in glob.glob(os.path.join(repodir, 'icons', '*.png')):
 279             resize_icon(iconpath)
 280
 281 def scan_apks(apps, apkcache, repodir, knownapks):
 282     """Scan the apks in the given repo directory.
 283
 284     This also extracts the icons.
 285
 286     :param apps: list of all applications, as per metadata.read_metadata
 287     :param apkcache: current apk cache information
 288     :param repodir: repo directory to scan
 289     :param knownapks: known apks info
 290     :returns: (apks, cachechanged) where apks is a list of apk information,
 291               and cachechanged is True if the apkcache got changed.
 292     """
 293
 294     cachechanged = False
 295
 296     icon_dir = os.path.join(repodir ,'icons')
 297     # Delete and re-create the icon directory...
 298     if options.clean and os.path.exists(icon_dir):
 299         shutil.rmtree(icon_dir)
 300     if not os.path.exists(icon_dir):
 301         os.makedirs(icon_dir)
 302     apks = []
 303     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 304     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 305     vername_pat = re.compile(".*versionName='([^']*)'.*")
 306     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 307     icon_pat = re.compile(".*icon='([^']+?)'.*")
 308     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 309     string_pat = re.compile(".*'([^']*)'.*")
 310     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 311
 312         apkfilename = apkfile[len(repodir) + 1:]
 313         if ' ' in apkfilename:
 314             print "No spaces in APK filenames!"
 315             sys.exit(1)
 316
 317         if apkfilename in apkcache:
 318             if options.verbose:
 319                 print "Reading " + apkfilename + " from cache"
 320             thisinfo = apkcache[apkfilename]
 321
 322         else:
 323
 324             if not options.quiet:
 325                 print "Processing " + apkfilename
 326             thisinfo = {}
 327             thisinfo['apkname'] = apkfilename
 328             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 329             if os.path.exists(os.path.join(repodir, srcfilename)):
 330                 thisinfo['srcname'] = srcfilename
 331             thisinfo['size'] = os.path.getsize(apkfile)
 332             thisinfo['permissions'] = []
 333             thisinfo['features'] = []
 334             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 335                                   'dump', 'badging', apkfile],
 336                                  stdout=subprocess.PIPE)
 337             output = p.communicate()[0]
 338             if options.verbose:
 339                 print output
 340             if p.returncode != 0:
 341                 print "ERROR: Failed to get apk information"
 342                 sys.exit(1)
 343             for line in output.splitlines():
 344                 if line.startswith("package:"):
 345                     try:
 346                         thisinfo['id'] = re.match(name_pat, line).group(1)
 347                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 348                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 349                     except Exception, e:
 350                         print "Package matching failed: " + str(e)
 351                         print "Line was: " + line
 352                         sys.exit(1)
 353                 elif line.startswith("application:"):
 354                     thisinfo['name'] = re.match(label_pat, line).group(1)
 355                     match = re.match(icon_pat, line)
 356                     if match:
 357                         thisinfo['iconsrc'] = match.group(1)
 358                 elif line.startswith("sdkVersion:"):
 359                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 360                 elif line.startswith("native-code:"):
 361                     thisinfo['nativecode'] = []
 362                     for arch in line[13:].split(' '):
 363                         thisinfo['nativecode'].append(arch[1:-1])
 364                 elif line.startswith("uses-permission:"):
 365                     perm = re.match(string_pat, line).group(1)
 366                     if perm.startswith("android.permission."):
 367                         perm = perm[19:]
 368                     thisinfo['permissions'].append(perm)
 369                 elif line.startswith("uses-feature:"):
 370                     perm = re.match(string_pat, line).group(1)
 371                     #Filter out this, it's only added with the latest SDK tools and
 372                     #causes problems for lots of apps.
 373                     if (perm != "android.hardware.screen.portrait" and
 374                         perm != "android.hardware.screen.landscape"):
 375                         if perm.startswith("android.feature."):
 376                             perm = perm[16:]
 377                         thisinfo['features'].append(perm)
 378
 379             if not 'sdkversion' in thisinfo:
 380                 print "  WARNING: no SDK version information found"
 381                 thisinfo['sdkversion'] = 0
 382
 383             # Check for debuggable apks...
 384             if common.isApkDebuggable(apkfile, config):
 385                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 386
 387             # Calculate the sha256...
 388             sha = hashlib.sha256()
 389             with open(apkfile, 'rb') as f:
 390                 while True:
 391                     t = f.read(1024)
 392                     if len(t) == 0:
 393                         break
 394                     sha.update(t)
 395                 thisinfo['sha256'] = sha.hexdigest()
 396
 397             # Get the signature (or md5 of, to be precise)...
 398             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 399             if not os.path.exists(getsig_dir + "/getsig.class"):
 400                 print "ERROR: getsig.class not found. To fix:"
 401                 print "\tcd " + getsig_dir
 402                 print "\t./make.sh"
 403                 sys.exit(1)
 404             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 405                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 406             output = p.communicate()[0]
 407             if options.verbose:
 408                 print output
 409             if p.returncode != 0 or not output.startswith('Result:'):
 410                 print "ERROR: Failed to get apk signature"
 411                 sys.exit(1)
 412             thisinfo['sig'] = output[7:].strip()
 413
 414             # Extract the icon file...
 415             if 'iconsrc' in thisinfo:
 416                 apk = zipfile.ZipFile(apkfile, 'r')
 417                 thisinfo['icon'] = (thisinfo['id'] + '.' +
 418                     str(thisinfo['versioncode']) + '.png')
 419                 iconpath = os.path.join(icon_dir, thisinfo['icon'])
 420                 try:
 421                     iconfile = open(iconpath, 'wb')
 422                     iconfile.write(apk.read(thisinfo['iconsrc']))
 423                     iconfile.close()
 424                 except:
 425                     print "WARNING: Error retrieving icon file"
 426                 apk.close()
 427
 428                 resize_icon(iconpath)
 429
 430             # Record in known apks, getting the added date at the same time..
 431             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 432             if added:
 433                 thisinfo['added'] = added
 434
 435             apkcache[apkfilename] = thisinfo
 436             cachechanged = True
 437
 438         apks.append(thisinfo)
 439
 440     return apks, cachechanged
 441
 442
 443 repo_pubkey_fingerprint = None
 444
 445 def make_index(apps, apks, repodir, archive, categories):
 446     """Make a repo index.
 447
 448     :param apps: fully populated apps list
 449     :param apks: full populated apks list
 450     :param repodir: the repo directory
 451     :param archive: True if this is the archive repo, False if it's the
 452                     main one.
 453     :param categories: list of categories
 454     """
 455
 456     doc = Document()
 457
 458     def addElement(name, value, doc, parent):
 459         el = doc.createElement(name)
 460         el.appendChild(doc.createTextNode(value))
 461         parent.appendChild(el)
 462     def addElementCDATA(name, value, doc, parent):
 463         el = doc.createElement(name)
 464         el.appendChild(doc.createCDATASection(value))
 465         parent.appendChild(el)
 466
 467     root = doc.createElement("fdroid")
 468     doc.appendChild(root)
 469
 470     repoel = doc.createElement("repo")
 471     if archive:
 472         repoel.setAttribute("name", config['archive_name'])
 473         if config['repo_maxage'] != 0:
 474             repoel.setAttribute("maxage", str(config['repo_maxage']))
 475         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 476         repoel.setAttribute("url", config['archive_url'])
 477         addElement('description', config['archive_description'], doc, repoel)
 478     else:
 479         repoel.setAttribute("name", config['repo_name'])
 480         if config['repo_maxage'] != 0:
 481             repoel.setAttribute("maxage", str(config['repo_maxage']))
 482         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 483         repoel.setAttribute("url", config['repo_url'])
 484         addElement('description', config['repo_description'], doc, repoel)
 485     repoel.setAttribute("timestamp", str(int(time.time())))
 486
 487     if config['repo_keyalias'] is not None:
 488
 489         # Generate a certificate fingerprint the same way keytool does it
 490         # (but with slightly different formatting)
 491         def cert_fingerprint(data):
 492             digest = hashlib.sha1(data).digest()
 493             ret = []
 494             for i in range(4):
 495                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 496             return " ".join(ret)
 497
 498         def extract_pubkey():
 499             p = subprocess.Popen(['keytool', '-exportcert',
 500                                   '-alias', config['repo_keyalias'],
 501                                   '-keystore', config['keystore'],
 502                                   '-storepass', config['keystorepass']],
 503                                  stdout=subprocess.PIPE)
 504             cert = p.communicate()[0]
 505             if p.returncode != 0:
 506                 print "ERROR: Failed to get repo pubkey"
 507                 sys.exit(1)
 508             global repo_pubkey_fingerprint
 509             repo_pubkey_fingerprint = cert_fingerprint(cert)
 510             return "".join("%02x" % ord(b) for b in cert)
 511
 512         repoel.setAttribute("pubkey", extract_pubkey())
 513
 514     root.appendChild(repoel)
 515
 516     for app in apps:
 517
 518         if app['Disabled'] is not None:
 519             continue
 520
 521         # Get a list of the apks for this app...
 522         apklist = []
 523         for apk in apks:
 524             if apk['id'] == app['id']:
 525                 apklist.append(apk)
 526
 527         if len(apklist) == 0:
 528             continue
 529
 530         apel = doc.createElement("application")
 531         apel.setAttribute("id", app['id'])
 532         root.appendChild(apel)
 533
 534         addElement('id', app['id'], doc, apel)
 535         if 'added' in app:
 536             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 537         if 'lastupdated' in app:
 538             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 539         addElement('name', app['Name'], doc, apel)
 540         addElement('summary', app['Summary'], doc, apel)
 541         if app['icon'] is not None:
 542             addElement('icon', app['icon'], doc, apel)
 543         def linkres(link):
 544             for app in apps:
 545                 if app['id'] == link:
 546                     return ("fdroid.app:" + link, app['Name'])
 547             raise MetaDataException("Cannot resolve app id " + link)
 548         addElement('desc',
 549                 metadata.description_html(app['Description'], linkres), doc, apel)
 550         addElement('license', app['License'], doc, apel)
 551         if 'Categories' in app:
 552             appcategories = [c.strip() for c in app['Categories'].split(',')]
 553             addElement('categories', ','.join(appcategories), doc, apel)
 554             # We put the first (primary) category in LAST, which will have
 555             # the desired effect of making clients that only understand one
 556             # category see that one.
 557             addElement('category', appcategories[0], doc, apel)
 558         addElement('web', app['Web Site'], doc, apel)
 559         addElement('source', app['Source Code'], doc, apel)
 560         addElement('tracker', app['Issue Tracker'], doc, apel)
 561         if app['Donate'] is not None:
 562             addElement('donate', app['Donate'], doc, apel)
 563         if app['Bitcoin'] is not None:
 564             addElement('bitcoin', app['Bitcoin'], doc, apel)
 565         if app['Litecoin'] is not None:
 566             addElement('litecoin', app['Litecoin'], doc, apel)
 567         if app['FlattrID'] is not None:
 568             addElement('flattr', app['FlattrID'], doc, apel)
 569
 570         # These elements actually refer to the current version (i.e. which
 571         # one is recommended. They are historically mis-named, and need
 572         # changing, but stay like this for now to support existing clients.
 573         addElement('marketversion', app['Current Version'], doc, apel)
 574         addElement('marketvercode', app['Current Version Code'], doc, apel)
 575
 576         if app['AntiFeatures']:
 577             af = app['AntiFeatures'].split(',')
 578             # TODO: Temporarily not including UpstreamNonFree in the index,
 579             # because current F-Droid clients do not understand it, and also
 580             # look ugly when they encounter an unknown antifeature. This
 581             # filtering can be removed in time...
 582             if 'UpstreamNonFree' in af:
 583                 af.remove('UpstreamNonFree')
 584             if af:
 585                 addElement('antifeatures', ','.join(af), doc, apel)
 586         if app['Provides']:
 587             pv = app['Provides'].split(',')
 588             addElement('provides', ','.join(pv), doc, apel)
 589         if app['Requires Root']:
 590             addElement('requirements', 'root', doc, apel)
 591
 592         # Sort the apk list into version order, just so the web site
 593         # doesn't have to do any work by default...
 594         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 595
 596         # Check for duplicates - they will make the client unhappy...
 597         for i in range(len(apklist) - 1):
 598             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 599                 print "ERROR - duplicate versions"
 600                 print apklist[i]['apkname']
 601                 print apklist[i+1]['apkname']
 602                 sys.exit(1)
 603
 604         for apk in apklist:
 605             apkel = doc.createElement("package")
 606             apel.appendChild(apkel)
 607             addElement('version', apk['version'], doc, apkel)
 608             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 609             addElement('apkname', apk['apkname'], doc, apkel)
 610             if 'srcname' in apk:
 611                 addElement('srcname', apk['srcname'], doc, apkel)
 612             for hash_type in ['sha256']:
 613                 if not hash_type in apk:
 614                     continue
 615                 hashel = doc.createElement("hash")
 616                 hashel.setAttribute("type", hash_type)
 617                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 618                 apkel.appendChild(hashel)
 619             addElement('sig', apk['sig'], doc, apkel)
 620             addElement('size', str(apk['size']), doc, apkel)
 621             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 622             if 'added' in apk:
 623                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 624             if app['Requires Root']:
 625                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 626                     apk['permissions'].append('ACCESS_SUPERUSER')
 627
 628             if len(apk['permissions']) > 0:
 629                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 630             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 631                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 632             if len(apk['features']) > 0:
 633                 addElement('features', ','.join(apk['features']), doc, apkel)
 634
 635     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 636     if options.pretty:
 637         output = doc.toprettyxml()
 638     else:
 639         output = doc.toxml()
 640     of.write(output)
 641     of.close()
 642
 643     if config['repo_keyalias'] is not None:
 644
 645         if not options.quiet:
 646             print "Creating signed index."
 647             print "Key fingerprint:", repo_pubkey_fingerprint
 648
 649         #Create a jar of the index...
 650         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 651             cwd=repodir, stdout=subprocess.PIPE)
 652         output = p.communicate()[0]
 653         if options.verbose:
 654             print output
 655         if p.returncode != 0:
 656             print "ERROR: Failed to create jar file"
 657             sys.exit(1)
 658
 659         # Sign the index...
 660         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 661             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 662             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 663             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 664         output = p.communicate()[0]
 665         if p.returncode != 0:
 666             print "Failed to sign index"
 667             print output
 668             sys.exit(1)
 669         if options.verbose:
 670             print output
 671
 672     # Copy the repo icon into the repo directory...
 673     icon_dir=os.path.join(repodir ,'icons')
 674     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 675     shutil.copyfile(config['repo_icon'], iconfilename)
 676
 677     # Write a category list in the repo to allow quick access...
 678     catdata = ''
 679     for cat in categories:
 680         catdata += cat + '\n'
 681     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 682     f.write(catdata)
 683     f.close()
 684
 685
 686
 687 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 688
 689     for app in apps:
 690
 691         # Get a list of the apks for this app...
 692         apklist = []
 693         for apk in apks:
 694             if apk['id'] == app['id']:
 695                 apklist.append(apk)
 696
 697         # Sort the apk list into version order...
 698         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 699
 700         if app['Archive Policy']:
 701             keepversions = int(app['Archive Policy'][:-9])
 702         else:
 703             keepversions = defaultkeepversions
 704
 705         if len(apklist) > keepversions:
 706             for apk in apklist[keepversions:]:
 707                 print "Moving " + apk['apkname'] + " to archive"
 708                 shutil.move(os.path.join(repodir, apk['apkname']),
 709                     os.path.join(archivedir, apk['apkname']))
 710                 if 'srcname' in apk:
 711                     shutil.move(os.path.join(repodir, apk['srcname']),
 712                         os.path.join(archivedir, apk['srcname']))
 713                 apks.remove(apk)
 714
 715
 716 config = None
 717 options = None
 718
 719 def main():
 720
 721     global config, options
 722
 723     # Parse command line...
 724     parser = OptionParser()
 725     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 726                       help="Create skeleton metadata files that are missing")
 727     parser.add_option("-v", "--verbose", action="store_true", default=False,
 728                       help="Spew out even more information than normal")
 729     parser.add_option("-q", "--quiet", action="store_true", default=False,
 730                       help="No output, except for warnings and errors")
 731     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 732                       help="Report on build data status")
 733     parser.add_option("-i", "--interactive", default=False, action="store_true",
 734                       help="Interactively ask about things that need updating.")
 735     parser.add_option("-I", "--icons", action="store_true", default=False,
 736                       help="Resize all the icons exceeding the max pixel size and exit")
 737     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 738                       help="Specify editor to use in interactive mode. Default "+
 739                           "is /etc/alternatives/editor")
 740     parser.add_option("-w", "--wiki", default=False, action="store_true",
 741                       help="Update the wiki")
 742     parser.add_option("", "--pretty", action="store_true", default=False,
 743                       help="Produce human-readable index.xml")
 744     parser.add_option("--clean", action="store_true", default=False,
 745                       help="Clean update - don't uses caches, reprocess all apks")
 746     (options, args) = parser.parse_args()
 747
 748     config = common.read_config(options)
 749
 750     repodirs = ['repo']
 751     if config['archive_older'] != 0:
 752         repodirs.append('archive')
 753         if not os.path.exists('archive'):
 754             os.mkdir('archive')
 755
 756     if options.icons:
 757         resize_all_icons(repodirs)
 758         sys.exit(0)
 759
 760     # Get all apps...
 761     apps = metadata.read_metadata()
 762
 763     # Generate a list of categories...
 764     categories = []
 765     for app in apps:
 766         cats = app['Categories'].split(',')
 767         for cat in cats:
 768             if cat not in categories:
 769                 categories.append(cat)
 770
 771     # Read known apks data (will be updated and written back when we've finished)
 772     knownapks = common.KnownApks()
 773
 774     # Gather information about all the apk files in the repo directory, using
 775     # cached data if possible.
 776     apkcachefile = os.path.join('tmp', 'apkcache')
 777     if not options.clean and os.path.exists(apkcachefile):
 778         with open(apkcachefile, 'rb') as cf:
 779             apkcache = pickle.load(cf)
 780     else:
 781         apkcache = {}
 782     cachechanged = False
 783
 784     delete_disabled_builds(apps, apkcache, repodirs)
 785
 786     # Scan all apks in the main repo
 787     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 788     if cc:
 789         cachechanged = True
 790
 791     # Scan the archive repo for apks as well
 792     if len(repodirs) > 1:
 793         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 794         if cc:
 795             cachechanged = True
 796     else:
 797         archapks = []
 798
 799     # Some information from the apks needs to be applied up to the application
 800     # level. When doing this, we use the info from the most recent version's apk.
 801     # We deal with figuring out when the app was added and last updated at the
 802     # same time.
 803     for app in apps:
 804         bestver = 0
 805         added = None
 806         lastupdated = None
 807         for apk in apks + archapks:
 808             if apk['id'] == app['id']:
 809                 if apk['versioncode'] > bestver:
 810                     bestver = apk['versioncode']
 811                     bestapk = apk
 812
 813                 if 'added' in apk:
 814                     if not added or apk['added'] < added:
 815                         added = apk['added']
 816                     if not lastupdated or apk['added'] > lastupdated:
 817                         lastupdated = apk['added']
 818
 819         if added:
 820             app['added'] = added
 821         else:
 822             if options.verbose:
 823                 print "WARNING: Don't know when " + app['id'] + " was added"
 824         if lastupdated:
 825             app['lastupdated'] = lastupdated
 826         else:
 827             if options.verbose:
 828                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 829
 830         if bestver == 0:
 831             if app['Name'] is None:
 832                 app['Name'] = app['id']
 833             app['icon'] = None
 834             if options.verbose and app['Disabled'] is None:
 835                 print "WARNING: Application " + app['id'] + " has no packages"
 836         else:
 837             if app['Name'] is None:
 838                 app['Name'] = bestapk['name']
 839             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 840
 841     # Sort the app list by name, then the web site doesn't have to by default.
 842     # (we had to wait until we'd scanned the apks to do this, because mostly the
 843     # name comes from there!)
 844     apps = sorted(apps, key=lambda app: app['Name'].upper())
 845
 846     # Generate warnings for apk's with no metadata (or create skeleton
 847     # metadata files, if requested on the command line)
 848     for apk in apks:
 849         found = False
 850         for app in apps:
 851             if app['id'] == apk['id']:
 852                 found = True
 853                 break
 854         if not found:
 855             if options.createmeta:
 856                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 857                 f.write("License:Unknown\n")
 858                 f.write("Web Site:\n")
 859                 f.write("Source Code:\n")
 860                 f.write("Issue Tracker:\n")
 861                 f.write("Summary:" + apk['name'] + "\n")
 862                 f.write("Description:\n")
 863                 f.write(apk['name'] + "\n")
 864                 f.write(".\n")
 865                 f.close()
 866                 print "Generated skeleton metadata for " + apk['id']
 867             else:
 868                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 869                 print "       " + apk['name'] + " - " + apk['version']
 870
 871     if len(repodirs) > 1:
 872         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 873
 874     # Make the index for the main repo...
 875     make_index(apps, apks, repodirs[0], False, categories)
 876
 877     # If there's an archive repo,  make the index for it. We already scanned it
 878     # earlier on.
 879     if len(repodirs) > 1:
 880         make_index(apps, archapks, repodirs[1], True, categories)
 881
 882     if config['update_stats']:
 883
 884         # Update known apks info...
 885         knownapks.writeifchanged()
 886
 887         # Generate latest apps data for widget
 888         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 889             data = ''
 890             for line in file(os.path.join('stats', 'latestapps.txt')):
 891                 appid = line.rstrip()
 892                 data += appid + "\t"
 893                 for app in apps:
 894                     if app['id'] == appid:
 895                         data += app['Name'] + "\t"
 896                         if app['icon'] is not None:
 897                             data += app['icon'] + "\t"
 898                         data += app['License'] + "\n"
 899                         break
 900             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
 901             f.write(data)
 902             f.close()
 903
 904     if cachechanged:
 905         with open(apkcachefile, 'wb') as cf:
 906             pickle.dump(apkcache, cf)
 907
 908     # Update the wiki...
 909     if options.wiki:
 910         update_wiki(apps, apks + archapks)
 911
 912     print "Finished."
 913
 914 if __name__ == "__main__":
 915     main()
 916