fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return [480, 320, 240, 160, 120]
  40
  41 def launcher_size(density):
  42     return (density * 48) / 160
  43
  44 def icon_dens_label(density):
  45     return "icon-%s" % density
  46
  47 def get_icon_dir(density):
  48     return os.path.join(repodir, "icons-%s" % density)
  49
  50 def get_icon_dirs(repodir):
  51     for density in get_densities():
  52         yield get_icon_dir(density)
  53
  54 def update_wiki(apps, apks):
  55     """Update the wiki
  56
  57     :param apps: fully populated list of all applications
  58     :param apks: all apks, except...
  59     """
  60     print "Updating wiki"
  61     wikicat = 'Apps'
  62     wikiredircat = 'App Redirects'
  63     import mwclient
  64     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  65             path=config['wiki_path'])
  66     site.login(config['wiki_user'], config['wiki_password'])
  67     generated_pages = {}
  68     generated_redirects = {}
  69     for app in apps:
  70         wikidata = ''
  71         if app['Disabled']:
  72             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  73         if app['AntiFeatures']:
  74             for af in app['AntiFeatures'].split(','):
  75                 wikidata += '{{AntiFeature|' + af + '}}\n'
  76         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  77                 app['id'],
  78                 app['Name'],
  79                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  80                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  81                 app['Source Code'],
  82                 app['Issue Tracker'],
  83                 app['Web Site'],
  84                 app['Donate'],
  85                 app['FlattrID'],
  86                 app['Bitcoin'],
  87                 app['Litecoin'],
  88                 app['Dogecoin'],
  89                 app['License'],
  90                 app.get('Requires Root', 'No'))
  91
  92         if app['Provides']:
  93             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  94
  95         wikidata += app['Summary']
  96         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  97
  98         wikidata += "=Description=\n"
  99         wikidata += metadata.description_wiki(app['Description']) + "\n"
 100
 101         wikidata += "=Maintainer Notes=\n"
 102         if 'Maintainer Notes' in app:
 103             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 104         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 105
 106         # Get a list of all packages for this application...
 107         apklist = []
 108         gotcurrentver = False
 109         cantupdate = False
 110         buildfails = False
 111         for apk in apks:
 112             if apk['id'] == app['id']:
 113                 if str(apk['versioncode']) == app['Current Version Code']:
 114                     gotcurrentver = True
 115                 apklist.append(apk)
 116         # Include ones we can't build, as a special case...
 117         for thisbuild in app['builds']:
 118             if 'disable' in thisbuild:
 119                 if thisbuild['vercode'] == app['Current Version Code']:
 120                     cantupdate = True
 121                 apklist.append({
 122                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 123                         'versioncode': int(thisbuild['vercode']),
 124                         'version': thisbuild['version'],
 125                         'buildproblem': thisbuild['disable']
 126                     })
 127             else:
 128                 builtit = False
 129                 for apk in apklist:
 130                     if apk['versioncode'] == int(thisbuild['vercode']):
 131                         builtit = True
 132                         break
 133                 if not builtit:
 134                     buildfails = True
 135                     apklist.append({
 136                             'versioncode': int(thisbuild['vercode']),
 137                             'version': thisbuild['version'],
 138                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 139                         })
 140         # Sort with most recent first...
 141         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 142
 143         wikidata += "=Versions=\n"
 144         if len(apklist) == 0:
 145             wikidata += "We currently have no versions of this app available."
 146         elif not gotcurrentver:
 147             wikidata += "We don't have the current version of this app."
 148         else:
 149             wikidata += "We have the current version of this app."
 150         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 151         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 152         if len(app['No Source Since']) > 0:
 153             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 154         if len(app['Current Version']) > 0:
 155             wikidata += "The current (recommended) version is " + app['Current Version']
 156             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 157         validapks = 0
 158         for apk in apklist:
 159             wikidata += "==" + apk['version'] + "==\n"
 160
 161             if 'buildproblem' in apk:
 162                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 163             else:
 164                 validapks += 1
 165                 wikidata += "This version is built and signed by "
 166                 if 'srcname' in apk:
 167                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 168                 else:
 169                     wikidata += "the original developer.\n\n"
 170             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 171
 172         wikidata += '\n[[Category:' + wikicat + ']]\n'
 173         if len(app['No Source Since']) > 0:
 174             wikidata += '\n[[Category:Apps missing source code]]\n'
 175         elif validapks == 0 and not app['Disabled']:
 176             wikidata += '\n[[Category:Apps with no packages]]\n'
 177         elif cantupdate and not app['Disabled']:
 178             wikidata += "\n[[Category:Apps we can't update]]\n"
 179         elif buildfails and not app['Disabled']:
 180             wikidata += "\n[[Category:Apps with failing builds]]\n"
 181         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 182             wikidata += '\n[[Category:Apps to Update]]\n'
 183         if app['Disabled']:
 184             wikidata += '\n[[Category:Apps that are disabled]]\n'
 185         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 186             wikidata += '\n[[Category:Apps with no update check]]\n'
 187
 188         # We can't have underscores in the page name, even if they're in
 189         # the package ID, because MediaWiki messes with them...
 190         pagename = app['id'].replace('_', ' ')
 191
 192         # Drop a trailing newline, because mediawiki is going to drop it anyway
 193         # and it we don't we'll think the page has changed when it hasn't...
 194         if wikidata.endswith('\n'):
 195             wikidata = wikidata[:-1]
 196
 197         generated_pages[pagename] = wikidata
 198
 199         # Make a redirect from the name to the ID too, unless there's
 200         # already an existing page with the name and it isn't a redirect.
 201         noclobber = False
 202         apppagename = app['Name'].replace('_', ' ')
 203         apppagename = apppagename.replace('{', '')
 204         apppagename = apppagename.replace('}', ' ')
 205         apppagename = apppagename.replace(':', ' ')
 206         # Drop double spaces caused mostly by replacing ':' above
 207         apppagename = apppagename.replace('  ', ' ')
 208         for expagename in site.allpages(prefix=apppagename,
 209                 filterredir='nonredirects', generator=False):
 210             if expagename == apppagename:
 211                 noclobber = True
 212         # Another reason not to make the redirect page is if the app name
 213         # is the same as it's ID, because that will overwrite the real page
 214         # with an redirect to itself! (Although it seems like an odd
 215         # scenario this happens a lot, e.g. where there is metadata but no
 216         # builds or binaries to extract a name from.
 217         if apppagename == pagename:
 218             noclobber = True
 219         if not noclobber:
 220             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 221
 222     for tcat, genp in [(wikicat, generated_pages),
 223             (wikiredircat, generated_redirects)]:
 224         catpages = site.Pages['Category:' + tcat]
 225         existingpages = []
 226         for page in catpages:
 227             existingpages.append(page.name)
 228             if page.name in genp:
 229                 pagetxt = page.edit()
 230                 if pagetxt != genp[page.name]:
 231                     print "Updating modified page " + page.name
 232                     page.save(genp[page.name], summary='Auto-updated')
 233                 else:
 234                     if options.verbose:
 235                         print "Page " + page.name + " is unchanged"
 236             else:
 237                 print "Deleting page " + page.name
 238                 page.delete('No longer published')
 239         for pagename, text in genp.items():
 240             if options.verbose:
 241                 print "Checking " + pagename
 242             if not pagename in existingpages:
 243                 print "Creating page " + pagename
 244                 try:
 245                     newpage = site.Pages[pagename]
 246                     newpage.save(text, summary='Auto-created')
 247                 except:
 248                     print "...FAILED to create page"
 249
 250     # Purge server cache to ensure counts are up to date
 251     site.pages['Repository Maintenance'].purge()
 252
 253 def delete_disabled_builds(apps, apkcache, repodirs):
 254     """Delete disabled build outputs.
 255
 256     :param apps: list of all applications, as per metadata.read_metadata
 257     :param apkcache: current apk cache information
 258     :param repodirs: the repo directories to process
 259     """
 260     for app in apps:
 261         for build in app['builds']:
 262             if 'disable' in build:
 263                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 264                 for repodir in repodirs:
 265                     apkpath = os.path.join(repodir, apkfilename)
 266                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 267                     for name in [apkpath, srcpath]:
 268                         if os.path.exists(name):
 269                             print "Deleting disabled build output " + apkfilename
 270                             os.remove(name)
 271                 if apkfilename in apkcache:
 272                     del apkcache[apkfilename]
 273
 274 def resize_icon(iconpath):
 275     try:
 276         im = Image.open(iconpath)
 277         if any(length > config['max_icon_size'] for length in im.size):
 278             print iconpath, "is too large:", im.size
 279             im.thumbnail((config['max_icon_size'], config['max_icon_size']),
 280                     Image.ANTIALIAS)
 281             print iconpath, "new size:", im.size
 282             im.save(iconpath, "PNG")
 283         else:
 284             if options.verbose:
 285                 print iconpath, "is small enough:", im.size
 286     except Exception,e:
 287         print "ERROR: Failed processing {0} - {1}".format(iconpath, e)
 288
 289 def resize_all_icons(repodirs):
 290     """Resize all icons that exceed the max size
 291
 292     :param apps: list of all applications, as per metadata.read_metadata
 293     :param repodirs: the repo directories to process
 294     """
 295     for repodir in repodirs:
 296         for iconpath in glob.glob(os.path.join(repodir, 'icons', '*.png')):
 297             resize_icon(iconpath)
 298
 299 def scan_apks(apps, apkcache, repodir, knownapks):
 300     """Scan the apks in the given repo directory.
 301
 302     This also extracts the icons.
 303
 304     :param apps: list of all applications, as per metadata.read_metadata
 305     :param apkcache: current apk cache information
 306     :param repodir: repo directory to scan
 307     :param knownapks: known apks info
 308     :returns: (apks, cachechanged) where apks is a list of apk information,
 309               and cachechanged is True if the apkcache got changed.
 310     """
 311
 312     cachechanged = False
 313
 314     icon_dir = os.path.join(repodir ,'icons')
 315     # Delete and re-create the icon directory...
 316     if options.clean and os.path.exists(icon_dir):
 317         shutil.rmtree(icon_dir)
 318     if not os.path.exists(icon_dir):
 319         os.makedirs(icon_dir)
 320     apks = []
 321     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 322     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 323     vername_pat = re.compile(".*versionName='([^']*)'.*")
 324     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 325     icon_pat = re.compile(".*icon='([^']+?)'.*")
 326     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 327     string_pat = re.compile(".*'([^']*)'.*")
 328     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 329
 330         apkfilename = apkfile[len(repodir) + 1:]
 331         if ' ' in apkfilename:
 332             print "No spaces in APK filenames!"
 333             sys.exit(1)
 334
 335         if apkfilename in apkcache:
 336             if options.verbose:
 337                 print "Reading " + apkfilename + " from cache"
 338             thisinfo = apkcache[apkfilename]
 339
 340         else:
 341
 342             if not options.quiet:
 343                 print "Processing " + apkfilename
 344             thisinfo = {}
 345             thisinfo['apkname'] = apkfilename
 346             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 347             if os.path.exists(os.path.join(repodir, srcfilename)):
 348                 thisinfo['srcname'] = srcfilename
 349             thisinfo['size'] = os.path.getsize(apkfile)
 350             thisinfo['permissions'] = []
 351             thisinfo['features'] = []
 352             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 353                                   'dump', 'badging', apkfile],
 354                                  stdout=subprocess.PIPE)
 355             output = p.communicate()[0]
 356             if options.verbose:
 357                 print output
 358             if p.returncode != 0:
 359                 print "ERROR: Failed to get apk information"
 360                 sys.exit(1)
 361             for line in output.splitlines():
 362                 if line.startswith("package:"):
 363                     try:
 364                         thisinfo['id'] = re.match(name_pat, line).group(1)
 365                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 366                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 367                     except Exception, e:
 368                         print "Package matching failed: " + str(e)
 369                         print "Line was: " + line
 370                         sys.exit(1)
 371                 elif line.startswith("application:"):
 372                     thisinfo['name'] = re.match(label_pat, line).group(1)
 373                     match = re.match(icon_pat, line)
 374                     if match:
 375                         thisinfo['iconsrc'] = match.group(1)
 376                 elif line.startswith("sdkVersion:"):
 377                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 378                 elif line.startswith("native-code:"):
 379                     thisinfo['nativecode'] = []
 380                     for arch in line[13:].split(' '):
 381                         thisinfo['nativecode'].append(arch[1:-1])
 382                 elif line.startswith("uses-permission:"):
 383                     perm = re.match(string_pat, line).group(1)
 384                     if perm.startswith("android.permission."):
 385                         perm = perm[19:]
 386                     thisinfo['permissions'].append(perm)
 387                 elif line.startswith("uses-feature:"):
 388                     perm = re.match(string_pat, line).group(1)
 389                     #Filter out this, it's only added with the latest SDK tools and
 390                     #causes problems for lots of apps.
 391                     if (perm != "android.hardware.screen.portrait" and
 392                         perm != "android.hardware.screen.landscape"):
 393                         if perm.startswith("android.feature."):
 394                             perm = perm[16:]
 395                         thisinfo['features'].append(perm)
 396
 397             if not 'sdkversion' in thisinfo:
 398                 print "  WARNING: no SDK version information found"
 399                 thisinfo['sdkversion'] = 0
 400
 401             # Check for debuggable apks...
 402             if common.isApkDebuggable(apkfile, config):
 403                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 404
 405             # Calculate the sha256...
 406             sha = hashlib.sha256()
 407             with open(apkfile, 'rb') as f:
 408                 while True:
 409                     t = f.read(1024)
 410                     if len(t) == 0:
 411                         break
 412                     sha.update(t)
 413                 thisinfo['sha256'] = sha.hexdigest()
 414
 415             # Get the signature (or md5 of, to be precise)...
 416             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 417             if not os.path.exists(getsig_dir + "/getsig.class"):
 418                 print "ERROR: getsig.class not found. To fix:"
 419                 print "\tcd " + getsig_dir
 420                 print "\t./make.sh"
 421                 sys.exit(1)
 422             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 423                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 424             output = p.communicate()[0]
 425             if options.verbose:
 426                 print output
 427             if p.returncode != 0 or not output.startswith('Result:'):
 428                 print "ERROR: Failed to get apk signature"
 429                 sys.exit(1)
 430             thisinfo['sig'] = output[7:].strip()
 431
 432             # Extract the icon file...
 433             if 'iconsrc' in thisinfo:
 434                 apk = zipfile.ZipFile(apkfile, 'r')
 435                 thisinfo['icon'] = (thisinfo['id'] + '.' +
 436                     str(thisinfo['versioncode']) + '.png')
 437                 iconpath = os.path.join(icon_dir, thisinfo['icon'])
 438                 try:
 439                     iconfile = open(iconpath, 'wb')
 440                     iconfile.write(apk.read(thisinfo['iconsrc']))
 441                     iconfile.close()
 442                 except:
 443                     print "WARNING: Error retrieving icon file"
 444                 apk.close()
 445
 446                 resize_icon(iconpath)
 447
 448             # Record in known apks, getting the added date at the same time..
 449             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 450             if added:
 451                 thisinfo['added'] = added
 452
 453             apkcache[apkfilename] = thisinfo
 454             cachechanged = True
 455
 456         apks.append(thisinfo)
 457
 458     return apks, cachechanged
 459
 460
 461 repo_pubkey_fingerprint = None
 462
 463 def make_index(apps, apks, repodir, archive, categories):
 464     """Make a repo index.
 465
 466     :param apps: fully populated apps list
 467     :param apks: full populated apks list
 468     :param repodir: the repo directory
 469     :param archive: True if this is the archive repo, False if it's the
 470                     main one.
 471     :param categories: list of categories
 472     """
 473
 474     doc = Document()
 475
 476     def addElement(name, value, doc, parent):
 477         el = doc.createElement(name)
 478         el.appendChild(doc.createTextNode(value))
 479         parent.appendChild(el)
 480     def addElementCDATA(name, value, doc, parent):
 481         el = doc.createElement(name)
 482         el.appendChild(doc.createCDATASection(value))
 483         parent.appendChild(el)
 484
 485     root = doc.createElement("fdroid")
 486     doc.appendChild(root)
 487
 488     repoel = doc.createElement("repo")
 489
 490     if archive:
 491         repoel.setAttribute("name", config['archive_name'])
 492         if config['repo_maxage'] != 0:
 493             repoel.setAttribute("maxage", str(config['repo_maxage']))
 494         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 495         repoel.setAttribute("url", config['archive_url'])
 496         addElement('description', config['archive_description'], doc, repoel)
 497
 498     else:
 499         repoel.setAttribute("name", config['repo_name'])
 500         if config['repo_maxage'] != 0:
 501             repoel.setAttribute("maxage", str(config['repo_maxage']))
 502         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 503         repoel.setAttribute("url", config['repo_url'])
 504         addElement('description', config['repo_description'], doc, repoel)
 505
 506     repoel.setAttribute("version", "10")
 507     repoel.setAttribute("timestamp", str(int(time.time())))
 508
 509     if config['repo_keyalias']:
 510
 511         # Generate a certificate fingerprint the same way keytool does it
 512         # (but with slightly different formatting)
 513         def cert_fingerprint(data):
 514             digest = hashlib.sha1(data).digest()
 515             ret = []
 516             for i in range(4):
 517                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 518             return " ".join(ret)
 519
 520         def extract_pubkey():
 521             p = subprocess.Popen(['keytool', '-exportcert',
 522                                   '-alias', config['repo_keyalias'],
 523                                   '-keystore', config['keystore'],
 524                                   '-storepass', config['keystorepass']],
 525                                  stdout=subprocess.PIPE)
 526             cert = p.communicate()[0]
 527             if p.returncode != 0:
 528                 print "ERROR: Failed to get repo pubkey"
 529                 sys.exit(1)
 530             global repo_pubkey_fingerprint
 531             repo_pubkey_fingerprint = cert_fingerprint(cert)
 532             return "".join("%02x" % ord(b) for b in cert)
 533
 534         repoel.setAttribute("pubkey", extract_pubkey())
 535
 536     root.appendChild(repoel)
 537
 538     for app in apps:
 539
 540         if app['Disabled'] is not None:
 541             continue
 542
 543         # Get a list of the apks for this app...
 544         apklist = []
 545         for apk in apks:
 546             if apk['id'] == app['id']:
 547                 apklist.append(apk)
 548
 549         if len(apklist) == 0:
 550             continue
 551
 552         apel = doc.createElement("application")
 553         apel.setAttribute("id", app['id'])
 554         root.appendChild(apel)
 555
 556         addElement('id', app['id'], doc, apel)
 557         if 'added' in app:
 558             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 559         if 'lastupdated' in app:
 560             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 561         addElement('name', app['Name'], doc, apel)
 562         addElement('summary', app['Summary'], doc, apel)
 563         if app['icon']:
 564             addElement('icon', app['icon'], doc, apel)
 565         def linkres(link):
 566             for app in apps:
 567                 if app['id'] == link:
 568                     return ("fdroid.app:" + link, app['Name'])
 569             raise MetaDataException("Cannot resolve app id " + link)
 570         addElement('desc',
 571                 metadata.description_html(app['Description'], linkres), doc, apel)
 572         addElement('license', app['License'], doc, apel)
 573         if 'Categories' in app:
 574             appcategories = [c.strip() for c in app['Categories'].split(',')]
 575             addElement('categories', ','.join(appcategories), doc, apel)
 576             # We put the first (primary) category in LAST, which will have
 577             # the desired effect of making clients that only understand one
 578             # category see that one.
 579             addElement('category', appcategories[0], doc, apel)
 580         addElement('web', app['Web Site'], doc, apel)
 581         addElement('source', app['Source Code'], doc, apel)
 582         addElement('tracker', app['Issue Tracker'], doc, apel)
 583         if app['Donate']:
 584             addElement('donate', app['Donate'], doc, apel)
 585         if app['Bitcoin']:
 586             addElement('bitcoin', app['Bitcoin'], doc, apel)
 587         if app['Litecoin']:
 588             addElement('litecoin', app['Litecoin'], doc, apel)
 589         if app['Dogecoin']:
 590             addElement('dogecoin', app['Dogecoin'], doc, apel)
 591         if app['FlattrID']:
 592             addElement('flattr', app['FlattrID'], doc, apel)
 593
 594         # These elements actually refer to the current version (i.e. which
 595         # one is recommended. They are historically mis-named, and need
 596         # changing, but stay like this for now to support existing clients.
 597         addElement('marketversion', app['Current Version'], doc, apel)
 598         addElement('marketvercode', app['Current Version Code'], doc, apel)
 599
 600         if app['AntiFeatures']:
 601             af = app['AntiFeatures'].split(',')
 602             # TODO: Temporarily not including UpstreamNonFree in the index,
 603             # because current F-Droid clients do not understand it, and also
 604             # look ugly when they encounter an unknown antifeature. This
 605             # filtering can be removed in time...
 606             if 'UpstreamNonFree' in af:
 607                 af.remove('UpstreamNonFree')
 608             if af:
 609                 addElement('antifeatures', ','.join(af), doc, apel)
 610         if app['Provides']:
 611             pv = app['Provides'].split(',')
 612             addElement('provides', ','.join(pv), doc, apel)
 613         if app['Requires Root']:
 614             addElement('requirements', 'root', doc, apel)
 615
 616         # Sort the apk list into version order, just so the web site
 617         # doesn't have to do any work by default...
 618         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 619
 620         # Check for duplicates - they will make the client unhappy...
 621         for i in range(len(apklist) - 1):
 622             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 623                 print "ERROR - duplicate versions"
 624                 print apklist[i]['apkname']
 625                 print apklist[i+1]['apkname']
 626                 sys.exit(1)
 627
 628         for apk in apklist:
 629             apkel = doc.createElement("package")
 630             apel.appendChild(apkel)
 631             addElement('version', apk['version'], doc, apkel)
 632             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 633             addElement('apkname', apk['apkname'], doc, apkel)
 634             if 'srcname' in apk:
 635                 addElement('srcname', apk['srcname'], doc, apkel)
 636             for hash_type in ['sha256']:
 637                 if not hash_type in apk:
 638                     continue
 639                 hashel = doc.createElement("hash")
 640                 hashel.setAttribute("type", hash_type)
 641                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 642                 apkel.appendChild(hashel)
 643             addElement('sig', apk['sig'], doc, apkel)
 644             addElement('size', str(apk['size']), doc, apkel)
 645             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 646             if 'added' in apk:
 647                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 648             if app['Requires Root']:
 649                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 650                     apk['permissions'].append('ACCESS_SUPERUSER')
 651
 652             if len(apk['permissions']) > 0:
 653                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 654             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 655                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 656             if len(apk['features']) > 0:
 657                 addElement('features', ','.join(apk['features']), doc, apkel)
 658
 659     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 660     if options.pretty:
 661         output = doc.toprettyxml()
 662     else:
 663         output = doc.toxml()
 664     of.write(output)
 665     of.close()
 666
 667     if config['repo_keyalias'] is not None:
 668
 669         if not options.quiet:
 670             print "Creating signed index."
 671             print "Key fingerprint:", repo_pubkey_fingerprint
 672
 673         #Create a jar of the index...
 674         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 675             cwd=repodir, stdout=subprocess.PIPE)
 676         output = p.communicate()[0]
 677         if options.verbose:
 678             print output
 679         if p.returncode != 0:
 680             print "ERROR: Failed to create jar file"
 681             sys.exit(1)
 682
 683         # Sign the index...
 684         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 685             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 686             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 687             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 688         output = p.communicate()[0]
 689         if p.returncode != 0:
 690             print "Failed to sign index"
 691             print output
 692             sys.exit(1)
 693         if options.verbose:
 694             print output
 695
 696     # Copy the repo icon into the repo directory...
 697     icon_dir = os.path.join(repodir ,'icons')
 698     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 699     shutil.copyfile(config['repo_icon'], iconfilename)
 700
 701     # Write a category list in the repo to allow quick access...
 702     catdata = ''
 703     for cat in categories:
 704         catdata += cat + '\n'
 705     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 706     f.write(catdata)
 707     f.close()
 708
 709
 710
 711 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 712
 713     for app in apps:
 714
 715         # Get a list of the apks for this app...
 716         apklist = []
 717         for apk in apks:
 718             if apk['id'] == app['id']:
 719                 apklist.append(apk)
 720
 721         # Sort the apk list into version order...
 722         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 723
 724         if app['Archive Policy']:
 725             keepversions = int(app['Archive Policy'][:-9])
 726         else:
 727             keepversions = defaultkeepversions
 728
 729         if len(apklist) > keepversions:
 730             for apk in apklist[keepversions:]:
 731                 print "Moving " + apk['apkname'] + " to archive"
 732                 shutil.move(os.path.join(repodir, apk['apkname']),
 733                     os.path.join(archivedir, apk['apkname']))
 734                 if 'srcname' in apk:
 735                     shutil.move(os.path.join(repodir, apk['srcname']),
 736                         os.path.join(archivedir, apk['srcname']))
 737                 apks.remove(apk)
 738
 739
 740 config = None
 741 options = None
 742
 743 def main():
 744
 745     global config, options
 746
 747     # Parse command line...
 748     parser = OptionParser()
 749     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 750                       help="Create skeleton metadata files that are missing")
 751     parser.add_option("-v", "--verbose", action="store_true", default=False,
 752                       help="Spew out even more information than normal")
 753     parser.add_option("-q", "--quiet", action="store_true", default=False,
 754                       help="No output, except for warnings and errors")
 755     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 756                       help="Report on build data status")
 757     parser.add_option("-i", "--interactive", default=False, action="store_true",
 758                       help="Interactively ask about things that need updating.")
 759     parser.add_option("-I", "--icons", action="store_true", default=False,
 760                       help="Resize all the icons exceeding the max pixel size and exit")
 761     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 762                       help="Specify editor to use in interactive mode. Default "+
 763                           "is /etc/alternatives/editor")
 764     parser.add_option("-w", "--wiki", default=False, action="store_true",
 765                       help="Update the wiki")
 766     parser.add_option("", "--pretty", action="store_true", default=False,
 767                       help="Produce human-readable index.xml")
 768     parser.add_option("--clean", action="store_true", default=False,
 769                       help="Clean update - don't uses caches, reprocess all apks")
 770     (options, args) = parser.parse_args()
 771
 772     config = common.read_config(options)
 773
 774     repodirs = ['repo']
 775     if config['archive_older'] != 0:
 776         repodirs.append('archive')
 777         if not os.path.exists('archive'):
 778             os.mkdir('archive')
 779
 780     if options.icons:
 781         resize_all_icons(repodirs)
 782         sys.exit(0)
 783
 784     # Get all apps...
 785     apps = metadata.read_metadata()
 786
 787     # Generate a list of categories...
 788     categories = []
 789     for app in apps:
 790         cats = app['Categories'].split(',')
 791         for cat in cats:
 792             if cat not in categories:
 793                 categories.append(cat)
 794
 795     # Read known apks data (will be updated and written back when we've finished)
 796     knownapks = common.KnownApks()
 797
 798     # Gather information about all the apk files in the repo directory, using
 799     # cached data if possible.
 800     apkcachefile = os.path.join('tmp', 'apkcache')
 801     if not options.clean and os.path.exists(apkcachefile):
 802         with open(apkcachefile, 'rb') as cf:
 803             apkcache = pickle.load(cf)
 804     else:
 805         apkcache = {}
 806     cachechanged = False
 807
 808     delete_disabled_builds(apps, apkcache, repodirs)
 809
 810     # Scan all apks in the main repo
 811     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 812     if cc:
 813         cachechanged = True
 814
 815     # Scan the archive repo for apks as well
 816     if len(repodirs) > 1:
 817         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 818         if cc:
 819             cachechanged = True
 820     else:
 821         archapks = []
 822
 823     # Some information from the apks needs to be applied up to the application
 824     # level. When doing this, we use the info from the most recent version's apk.
 825     # We deal with figuring out when the app was added and last updated at the
 826     # same time.
 827     for app in apps:
 828         bestver = 0
 829         added = None
 830         lastupdated = None
 831         for apk in apks + archapks:
 832             if apk['id'] == app['id']:
 833                 if apk['versioncode'] > bestver:
 834                     bestver = apk['versioncode']
 835                     bestapk = apk
 836
 837                 if 'added' in apk:
 838                     if not added or apk['added'] < added:
 839                         added = apk['added']
 840                     if not lastupdated or apk['added'] > lastupdated:
 841                         lastupdated = apk['added']
 842
 843         if added:
 844             app['added'] = added
 845         else:
 846             if options.verbose:
 847                 print "WARNING: Don't know when " + app['id'] + " was added"
 848         if lastupdated:
 849             app['lastupdated'] = lastupdated
 850         else:
 851             if options.verbose:
 852                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 853
 854         if bestver == 0:
 855             if app['Name'] is None:
 856                 app['Name'] = app['id']
 857             app['icon'] = None
 858             if options.verbose and app['Disabled'] is None:
 859                 print "WARNING: Application " + app['id'] + " has no packages"
 860         else:
 861             if app['Name'] is None:
 862                 app['Name'] = bestapk['name']
 863             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 864
 865     # Sort the app list by name, then the web site doesn't have to by default.
 866     # (we had to wait until we'd scanned the apks to do this, because mostly the
 867     # name comes from there!)
 868     apps = sorted(apps, key=lambda app: app['Name'].upper())
 869
 870     # Generate warnings for apk's with no metadata (or create skeleton
 871     # metadata files, if requested on the command line)
 872     for apk in apks:
 873         found = False
 874         for app in apps:
 875             if app['id'] == apk['id']:
 876                 found = True
 877                 break
 878         if not found:
 879             if options.createmeta:
 880                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 881                 f.write("License:Unknown\n")
 882                 f.write("Web Site:\n")
 883                 f.write("Source Code:\n")
 884                 f.write("Issue Tracker:\n")
 885                 f.write("Summary:" + apk['name'] + "\n")
 886                 f.write("Description:\n")
 887                 f.write(apk['name'] + "\n")
 888                 f.write(".\n")
 889                 f.close()
 890                 print "Generated skeleton metadata for " + apk['id']
 891             else:
 892                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 893                 print "       " + apk['name'] + " - " + apk['version']
 894
 895     if len(repodirs) > 1:
 896         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 897
 898     # Make the index for the main repo...
 899     make_index(apps, apks, repodirs[0], False, categories)
 900
 901     # If there's an archive repo,  make the index for it. We already scanned it
 902     # earlier on.
 903     if len(repodirs) > 1:
 904         make_index(apps, archapks, repodirs[1], True, categories)
 905
 906     if config['update_stats']:
 907
 908         # Update known apks info...
 909         knownapks.writeifchanged()
 910
 911         # Generate latest apps data for widget
 912         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 913             data = ''
 914             for line in file(os.path.join('stats', 'latestapps.txt')):
 915                 appid = line.rstrip()
 916                 data += appid + "\t"
 917                 for app in apps:
 918                     if app['id'] == appid:
 919                         data += app['Name'] + "\t"
 920                         if app['icon'] is not None:
 921                             data += app['icon'] + "\t"
 922                         data += app['License'] + "\n"
 923                         break
 924             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
 925             f.write(data)
 926             f.close()
 927
 928     if cachechanged:
 929         with open(apkcachefile, 'wb') as cf:
 930             pickle.dump(apkcache, cf)
 931
 932     # Update the wiki...
 933     if options.wiki:
 934         update_wiki(apps, apks + archapks)
 935
 936     print "Finished."
 937
 938 if __name__ == "__main__":
 939     main()
 940