fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['480', '320', '240', '160', '120']
  40
  41 def launcher_size(density):
  42     return (int(density) * 48) / 160
  43
  44 def icon_dens_label(density):
  45     return "icon-%s" % density
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         elif validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         elif cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         elif buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190
 191         # We can't have underscores in the page name, even if they're in
 192         # the package ID, because MediaWiki messes with them...
 193         pagename = app['id'].replace('_', ' ')
 194
 195         # Drop a trailing newline, because mediawiki is going to drop it anyway
 196         # and it we don't we'll think the page has changed when it hasn't...
 197         if wikidata.endswith('\n'):
 198             wikidata = wikidata[:-1]
 199
 200         generated_pages[pagename] = wikidata
 201
 202         # Make a redirect from the name to the ID too, unless there's
 203         # already an existing page with the name and it isn't a redirect.
 204         noclobber = False
 205         apppagename = app['Name'].replace('_', ' ')
 206         apppagename = apppagename.replace('{', '')
 207         apppagename = apppagename.replace('}', ' ')
 208         apppagename = apppagename.replace(':', ' ')
 209         # Drop double spaces caused mostly by replacing ':' above
 210         apppagename = apppagename.replace('  ', ' ')
 211         for expagename in site.allpages(prefix=apppagename,
 212                 filterredir='nonredirects', generator=False):
 213             if expagename == apppagename:
 214                 noclobber = True
 215         # Another reason not to make the redirect page is if the app name
 216         # is the same as it's ID, because that will overwrite the real page
 217         # with an redirect to itself! (Although it seems like an odd
 218         # scenario this happens a lot, e.g. where there is metadata but no
 219         # builds or binaries to extract a name from.
 220         if apppagename == pagename:
 221             noclobber = True
 222         if not noclobber:
 223             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 224
 225     for tcat, genp in [(wikicat, generated_pages),
 226             (wikiredircat, generated_redirects)]:
 227         catpages = site.Pages['Category:' + tcat]
 228         existingpages = []
 229         for page in catpages:
 230             existingpages.append(page.name)
 231             if page.name in genp:
 232                 pagetxt = page.edit()
 233                 if pagetxt != genp[page.name]:
 234                     print "Updating modified page " + page.name
 235                     page.save(genp[page.name], summary='Auto-updated')
 236                 else:
 237                     if options.verbose:
 238                         print "Page " + page.name + " is unchanged"
 239             else:
 240                 print "Deleting page " + page.name
 241                 page.delete('No longer published')
 242         for pagename, text in genp.items():
 243             if options.verbose:
 244                 print "Checking " + pagename
 245             if not pagename in existingpages:
 246                 print "Creating page " + pagename
 247                 try:
 248                     newpage = site.Pages[pagename]
 249                     newpage.save(text, summary='Auto-created')
 250                 except:
 251                     print "...FAILED to create page"
 252
 253     # Purge server cache to ensure counts are up to date
 254     site.pages['Repository Maintenance'].purge()
 255
 256 def delete_disabled_builds(apps, apkcache, repodirs):
 257     """Delete disabled build outputs.
 258
 259     :param apps: list of all applications, as per metadata.read_metadata
 260     :param apkcache: current apk cache information
 261     :param repodirs: the repo directories to process
 262     """
 263     for app in apps:
 264         for build in app['builds']:
 265             if 'disable' in build:
 266                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 267                 for repodir in repodirs:
 268                     apkpath = os.path.join(repodir, apkfilename)
 269                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 270                     for name in [apkpath, srcpath]:
 271                         if os.path.exists(name):
 272                             print "Deleting disabled build output " + apkfilename
 273                             os.remove(name)
 274                 if apkfilename in apkcache:
 275                     del apkcache[apkfilename]
 276
 277 def resize_icon(iconpath, density):
 278
 279     try:
 280         im = Image.open(iconpath)
 281         size = launcher_size(density)
 282
 283         if any(length > size for length in im.size):
 284             print iconpath, "is too large:", im.size
 285             im.thumbnail((size, size), Image.ANTIALIAS)
 286             print iconpath, "new size:", im.size
 287             im.save(iconpath, "PNG")
 288
 289         else:
 290             if options.verbose:
 291                 print iconpath, "is small enough:", im.size
 292
 293     except Exception,e:
 294         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 295
 296 def resize_all_icons(repodirs):
 297     """Resize all icons that exceed the max size
 298
 299     :param repodirs: the repo directories to process
 300     """
 301     for repodir in repodirs:
 302         for density in get_densities():
 303             icon_dir = get_icon_dir(repodir, density)
 304             icon_glob = os.path.join(icon_dir, '*.png')
 305             for iconpath in glob.glob(icon_glob):
 306                 resize_icon(iconpath, density)
 307
 308 def scan_apks(apps, apkcache, repodir, knownapks):
 309     """Scan the apks in the given repo directory.
 310
 311     This also extracts the icons.
 312
 313     :param apps: list of all applications, as per metadata.read_metadata
 314     :param apkcache: current apk cache information
 315     :param repodir: repo directory to scan
 316     :param knownapks: known apks info
 317     :returns: (apks, cachechanged) where apks is a list of apk information,
 318               and cachechanged is True if the apkcache got changed.
 319     """
 320
 321     cachechanged = False
 322
 323     icon_dirs = get_icon_dirs(repodir)
 324     for icon_dir in icon_dirs:
 325         if os.path.exists(icon_dir):
 326             if options.clean:
 327                 shutil.rmtree(icon_dir)
 328                 os.makedirs(icon_dir)
 329         else:
 330             os.makedirs(icon_dir)
 331
 332     apks = []
 333     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 334     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 335     vername_pat = re.compile(".*versionName='([^']*)'.*")
 336     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 337     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 338     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 339     string_pat = re.compile(".*'([^']*)'.*")
 340     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 341
 342         apkfilename = apkfile[len(repodir) + 1:]
 343         if ' ' in apkfilename:
 344             print "No spaces in APK filenames!"
 345             sys.exit(1)
 346
 347         if apkfilename in apkcache:
 348             if options.verbose:
 349                 print "Reading " + apkfilename + " from cache"
 350             thisinfo = apkcache[apkfilename]
 351
 352         else:
 353
 354             if not options.quiet:
 355                 print "Processing " + apkfilename
 356             thisinfo = {}
 357             thisinfo['apkname'] = apkfilename
 358             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 359             if os.path.exists(os.path.join(repodir, srcfilename)):
 360                 thisinfo['srcname'] = srcfilename
 361             thisinfo['size'] = os.path.getsize(apkfile)
 362             thisinfo['permissions'] = []
 363             thisinfo['features'] = []
 364             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 365                                   'dump', 'badging', apkfile],
 366                                  stdout=subprocess.PIPE)
 367             output = p.communicate()[0]
 368             if p.returncode != 0:
 369                 print "ERROR: Failed to get apk information"
 370                 sys.exit(1)
 371             for line in output.splitlines():
 372                 if line.startswith("package:"):
 373                     try:
 374                         thisinfo['id'] = re.match(name_pat, line).group(1)
 375                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 376                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 377                     except Exception, e:
 378                         print "Package matching failed: " + str(e)
 379                         print "Line was: " + line
 380                         sys.exit(1)
 381                 elif line.startswith("application:"):
 382                     thisinfo['name'] = re.match(label_pat, line).group(1)
 383                 elif line.startswith("application-icon-"):
 384                     match = re.match(icon_pat, line)
 385                     if match:
 386                         density = match.group(1)
 387                         path = match.group(2)
 388                         if 'icons_src' not in thisinfo:
 389                             thisinfo['icons_src'] = {}
 390                         thisinfo['icons_src'][density] = path
 391                 elif line.startswith("sdkVersion:"):
 392                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 393                 elif line.startswith("native-code:"):
 394                     thisinfo['nativecode'] = []
 395                     for arch in line[13:].split(' '):
 396                         thisinfo['nativecode'].append(arch[1:-1])
 397                 elif line.startswith("uses-permission:"):
 398                     perm = re.match(string_pat, line).group(1)
 399                     if perm.startswith("android.permission."):
 400                         perm = perm[19:]
 401                     thisinfo['permissions'].append(perm)
 402                 elif line.startswith("uses-feature:"):
 403                     perm = re.match(string_pat, line).group(1)
 404                     #Filter out this, it's only added with the latest SDK tools and
 405                     #causes problems for lots of apps.
 406                     if (perm != "android.hardware.screen.portrait" and
 407                         perm != "android.hardware.screen.landscape"):
 408                         if perm.startswith("android.feature."):
 409                             perm = perm[16:]
 410                         thisinfo['features'].append(perm)
 411
 412             if not 'sdkversion' in thisinfo:
 413                 print "  WARNING: no SDK version information found"
 414                 thisinfo['sdkversion'] = 0
 415
 416             # Check for debuggable apks...
 417             if common.isApkDebuggable(apkfile, config):
 418                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 419
 420             # Calculate the sha256...
 421             sha = hashlib.sha256()
 422             with open(apkfile, 'rb') as f:
 423                 while True:
 424                     t = f.read(1024)
 425                     if len(t) == 0:
 426                         break
 427                     sha.update(t)
 428                 thisinfo['sha256'] = sha.hexdigest()
 429
 430             # Get the signature (or md5 of, to be precise)...
 431             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 432             if not os.path.exists(getsig_dir + "/getsig.class"):
 433                 print "ERROR: getsig.class not found. To fix:"
 434                 print "\tcd " + getsig_dir
 435                 print "\t./make.sh"
 436                 sys.exit(1)
 437             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 438                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 439             output = p.communicate()[0]
 440             if options.verbose:
 441                 print output
 442             if p.returncode != 0 or not output.startswith('Result:'):
 443                 print "ERROR: Failed to get apk signature"
 444                 sys.exit(1)
 445             thisinfo['sig'] = output[7:].strip()
 446
 447             iconfilename = "%s.%s.png" % (
 448                     thisinfo['id'],
 449                     thisinfo['versioncode'])
 450
 451             # Extract the icon file...
 452             densities = get_densities()
 453             empty_densities = []
 454             for density in densities:
 455                 label = icon_dens_label(density)
 456                 if density not in thisinfo['icons_src']:
 457                     empty_densities.append(density)
 458                     continue
 459                 apk = zipfile.ZipFile(apkfile, 'r')
 460                 iconsrc = thisinfo['icons_src'][density]
 461                 icon_dir = get_icon_dir(repodir, density)
 462                 icondest = os.path.join(icon_dir, iconfilename)
 463
 464                 try:
 465                     iconfile = open(icondest, 'wb')
 466                     iconfile.write(apk.read(iconsrc))
 467                     iconfile.close()
 468                     thisinfo[label] = iconfilename
 469
 470                 except:
 471                     print "WARNING: Error retrieving icon file"
 472                     del thisinfo[label]
 473                     del thisinfo['icons_src'][density]
 474                     empty_densities.append(density)
 475
 476                 apk.close()
 477
 478                 resize_icon(icondest, density)
 479
 480             last_density = None
 481             # First try resizing down to not lose quality
 482             for density in densities:
 483                 if density not in empty_densities:
 484                     last_density = density
 485                     continue
 486                 if last_density is None:
 487                     continue
 488                 if options.verbose:
 489                     print "Density %s not available, resizing down from %s" % (
 490                             density, last_density)
 491
 492                 last_iconpath = os.path.join(
 493                         get_icon_dir(repodir, last_density), iconfilename)
 494                 iconpath = os.path.join(
 495                         get_icon_dir(repodir, density), iconfilename)
 496                 im = Image.open(last_iconpath)
 497                 size = launcher_size(density)
 498
 499                 im.thumbnail((size, size), Image.ANTIALIAS)
 500                 im.save(iconpath, "PNG")
 501                 empty_densities.remove(density)
 502
 503             last_density = None
 504             # Then just copy from the highest resolution available
 505             for density in reversed(densities):
 506                 if density not in empty_densities:
 507                     last_density = density
 508                     continue
 509                 if last_density is None:
 510                     continue
 511                 if options.verbose:
 512                     print "Density %s not available, copying from lower density %s" % (
 513                             density, last_density)
 514
 515                 shutil.copyfile(
 516                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 517                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 518
 519                 empty_densities.remove(density)
 520
 521             # Copy from icons-mdpi to icons since mdpi is the baseline density
 522             shutil.copyfile(
 523                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 524                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 525
 526             # Record in known apks, getting the added date at the same time..
 527             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 528             if added:
 529                 thisinfo['added'] = added
 530
 531             apkcache[apkfilename] = thisinfo
 532             cachechanged = True
 533
 534         apks.append(thisinfo)
 535
 536     return apks, cachechanged
 537
 538
 539 repo_pubkey_fingerprint = None
 540
 541 def make_index(apps, apks, repodir, archive, categories):
 542     """Make a repo index.
 543
 544     :param apps: fully populated apps list
 545     :param apks: full populated apks list
 546     :param repodir: the repo directory
 547     :param archive: True if this is the archive repo, False if it's the
 548                     main one.
 549     :param categories: list of categories
 550     """
 551
 552     doc = Document()
 553
 554     def addElement(name, value, doc, parent):
 555         el = doc.createElement(name)
 556         el.appendChild(doc.createTextNode(value))
 557         parent.appendChild(el)
 558     def addElementCDATA(name, value, doc, parent):
 559         el = doc.createElement(name)
 560         el.appendChild(doc.createCDATASection(value))
 561         parent.appendChild(el)
 562
 563     root = doc.createElement("fdroid")
 564     doc.appendChild(root)
 565
 566     repoel = doc.createElement("repo")
 567
 568     if archive:
 569         repoel.setAttribute("name", config['archive_name'])
 570         if config['repo_maxage'] != 0:
 571             repoel.setAttribute("maxage", str(config['repo_maxage']))
 572         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 573         repoel.setAttribute("url", config['archive_url'])
 574         addElement('description', config['archive_description'], doc, repoel)
 575
 576     else:
 577         repoel.setAttribute("name", config['repo_name'])
 578         if config['repo_maxage'] != 0:
 579             repoel.setAttribute("maxage", str(config['repo_maxage']))
 580         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 581         repoel.setAttribute("url", config['repo_url'])
 582         addElement('description', config['repo_description'], doc, repoel)
 583
 584     repoel.setAttribute("version", "10")
 585     repoel.setAttribute("timestamp", str(int(time.time())))
 586
 587     if config['repo_keyalias']:
 588
 589         # Generate a certificate fingerprint the same way keytool does it
 590         # (but with slightly different formatting)
 591         def cert_fingerprint(data):
 592             digest = hashlib.sha1(data).digest()
 593             ret = []
 594             for i in range(4):
 595                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 596             return " ".join(ret)
 597
 598         def extract_pubkey():
 599             p = subprocess.Popen(['keytool', '-exportcert',
 600                                   '-alias', config['repo_keyalias'],
 601                                   '-keystore', config['keystore'],
 602                                   '-storepass', config['keystorepass']],
 603                                  stdout=subprocess.PIPE)
 604             cert = p.communicate()[0]
 605             if p.returncode != 0:
 606                 print "ERROR: Failed to get repo pubkey"
 607                 sys.exit(1)
 608             global repo_pubkey_fingerprint
 609             repo_pubkey_fingerprint = cert_fingerprint(cert)
 610             return "".join("%02x" % ord(b) for b in cert)
 611
 612         repoel.setAttribute("pubkey", extract_pubkey())
 613
 614     root.appendChild(repoel)
 615
 616     for app in apps:
 617
 618         if app['Disabled'] is not None:
 619             continue
 620
 621         # Get a list of the apks for this app...
 622         apklist = []
 623         for apk in apks:
 624             if apk['id'] == app['id']:
 625                 apklist.append(apk)
 626
 627         if len(apklist) == 0:
 628             continue
 629
 630         apel = doc.createElement("application")
 631         apel.setAttribute("id", app['id'])
 632         root.appendChild(apel)
 633
 634         addElement('id', app['id'], doc, apel)
 635         if 'added' in app:
 636             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 637         if 'lastupdated' in app:
 638             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 639         addElement('name', app['Name'], doc, apel)
 640         addElement('summary', app['Summary'], doc, apel)
 641         if app['icon']:
 642             addElement('icon', app['icon'], doc, apel)
 643         def linkres(link):
 644             for app in apps:
 645                 if app['id'] == link:
 646                     return ("fdroid.app:" + link, app['Name'])
 647             raise MetaDataException("Cannot resolve app id " + link)
 648         addElement('desc',
 649                 metadata.description_html(app['Description'], linkres), doc, apel)
 650         addElement('license', app['License'], doc, apel)
 651         if 'Categories' in app:
 652             appcategories = [c.strip() for c in app['Categories'].split(',')]
 653             addElement('categories', ','.join(appcategories), doc, apel)
 654             # We put the first (primary) category in LAST, which will have
 655             # the desired effect of making clients that only understand one
 656             # category see that one.
 657             addElement('category', appcategories[0], doc, apel)
 658         addElement('web', app['Web Site'], doc, apel)
 659         addElement('source', app['Source Code'], doc, apel)
 660         addElement('tracker', app['Issue Tracker'], doc, apel)
 661         if app['Donate']:
 662             addElement('donate', app['Donate'], doc, apel)
 663         if app['Bitcoin']:
 664             addElement('bitcoin', app['Bitcoin'], doc, apel)
 665         if app['Litecoin']:
 666             addElement('litecoin', app['Litecoin'], doc, apel)
 667         if app['Dogecoin']:
 668             addElement('dogecoin', app['Dogecoin'], doc, apel)
 669         if app['FlattrID']:
 670             addElement('flattr', app['FlattrID'], doc, apel)
 671
 672         # These elements actually refer to the current version (i.e. which
 673         # one is recommended. They are historically mis-named, and need
 674         # changing, but stay like this for now to support existing clients.
 675         addElement('marketversion', app['Current Version'], doc, apel)
 676         addElement('marketvercode', app['Current Version Code'], doc, apel)
 677
 678         if app['AntiFeatures']:
 679             af = app['AntiFeatures'].split(',')
 680             # TODO: Temporarily not including UpstreamNonFree in the index,
 681             # because current F-Droid clients do not understand it, and also
 682             # look ugly when they encounter an unknown antifeature. This
 683             # filtering can be removed in time...
 684             if 'UpstreamNonFree' in af:
 685                 af.remove('UpstreamNonFree')
 686             if af:
 687                 addElement('antifeatures', ','.join(af), doc, apel)
 688         if app['Provides']:
 689             pv = app['Provides'].split(',')
 690             addElement('provides', ','.join(pv), doc, apel)
 691         if app['Requires Root']:
 692             addElement('requirements', 'root', doc, apel)
 693
 694         # Sort the apk list into version order, just so the web site
 695         # doesn't have to do any work by default...
 696         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 697
 698         # Check for duplicates - they will make the client unhappy...
 699         for i in range(len(apklist) - 1):
 700             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 701                 print "ERROR - duplicate versions"
 702                 print apklist[i]['apkname']
 703                 print apklist[i+1]['apkname']
 704                 sys.exit(1)
 705
 706         for apk in apklist:
 707             apkel = doc.createElement("package")
 708             apel.appendChild(apkel)
 709             addElement('version', apk['version'], doc, apkel)
 710             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 711             addElement('apkname', apk['apkname'], doc, apkel)
 712             if 'srcname' in apk:
 713                 addElement('srcname', apk['srcname'], doc, apkel)
 714             for hash_type in ['sha256']:
 715                 if not hash_type in apk:
 716                     continue
 717                 hashel = doc.createElement("hash")
 718                 hashel.setAttribute("type", hash_type)
 719                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 720                 apkel.appendChild(hashel)
 721             addElement('sig', apk['sig'], doc, apkel)
 722             addElement('size', str(apk['size']), doc, apkel)
 723             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 724             if 'added' in apk:
 725                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 726             if app['Requires Root']:
 727                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 728                     apk['permissions'].append('ACCESS_SUPERUSER')
 729
 730             if len(apk['permissions']) > 0:
 731                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 732             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 733                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 734             if len(apk['features']) > 0:
 735                 addElement('features', ','.join(apk['features']), doc, apkel)
 736
 737     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 738     if options.pretty:
 739         output = doc.toprettyxml()
 740     else:
 741         output = doc.toxml()
 742     of.write(output)
 743     of.close()
 744
 745     if config['repo_keyalias'] is not None:
 746
 747         if not options.quiet:
 748             print "Creating signed index."
 749             print "Key fingerprint:", repo_pubkey_fingerprint
 750
 751         #Create a jar of the index...
 752         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 753             cwd=repodir, stdout=subprocess.PIPE)
 754         output = p.communicate()[0]
 755         if options.verbose:
 756             print output
 757         if p.returncode != 0:
 758             print "ERROR: Failed to create jar file"
 759             sys.exit(1)
 760
 761         # Sign the index...
 762         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 763             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 764             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 765             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 766         output = p.communicate()[0]
 767         if p.returncode != 0:
 768             print "Failed to sign index"
 769             print output
 770             sys.exit(1)
 771         if options.verbose:
 772             print output
 773
 774     # Copy the repo icon into the repo directory...
 775     icon_dir = os.path.join(repodir ,'icons')
 776     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 777     shutil.copyfile(config['repo_icon'], iconfilename)
 778
 779     # Write a category list in the repo to allow quick access...
 780     catdata = ''
 781     for cat in categories:
 782         catdata += cat + '\n'
 783     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 784     f.write(catdata)
 785     f.close()
 786
 787
 788
 789 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 790
 791     for app in apps:
 792
 793         # Get a list of the apks for this app...
 794         apklist = []
 795         for apk in apks:
 796             if apk['id'] == app['id']:
 797                 apklist.append(apk)
 798
 799         # Sort the apk list into version order...
 800         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 801
 802         if app['Archive Policy']:
 803             keepversions = int(app['Archive Policy'][:-9])
 804         else:
 805             keepversions = defaultkeepversions
 806
 807         if len(apklist) > keepversions:
 808             for apk in apklist[keepversions:]:
 809                 print "Moving " + apk['apkname'] + " to archive"
 810                 shutil.move(os.path.join(repodir, apk['apkname']),
 811                     os.path.join(archivedir, apk['apkname']))
 812                 if 'srcname' in apk:
 813                     shutil.move(os.path.join(repodir, apk['srcname']),
 814                         os.path.join(archivedir, apk['srcname']))
 815                 apks.remove(apk)
 816
 817
 818 config = None
 819 options = None
 820
 821 def main():
 822
 823     global config, options
 824
 825     # Parse command line...
 826     parser = OptionParser()
 827     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 828                       help="Create skeleton metadata files that are missing")
 829     parser.add_option("-v", "--verbose", action="store_true", default=False,
 830                       help="Spew out even more information than normal")
 831     parser.add_option("-q", "--quiet", action="store_true", default=False,
 832                       help="No output, except for warnings and errors")
 833     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 834                       help="Report on build data status")
 835     parser.add_option("-i", "--interactive", default=False, action="store_true",
 836                       help="Interactively ask about things that need updating.")
 837     parser.add_option("-I", "--icons", action="store_true", default=False,
 838                       help="Resize all the icons exceeding the max pixel size and exit")
 839     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 840                       help="Specify editor to use in interactive mode. Default "+
 841                           "is /etc/alternatives/editor")
 842     parser.add_option("-w", "--wiki", default=False, action="store_true",
 843                       help="Update the wiki")
 844     parser.add_option("", "--pretty", action="store_true", default=False,
 845                       help="Produce human-readable index.xml")
 846     parser.add_option("--clean", action="store_true", default=False,
 847                       help="Clean update - don't uses caches, reprocess all apks")
 848     (options, args) = parser.parse_args()
 849
 850     config = common.read_config(options)
 851
 852     repodirs = ['repo']
 853     if config['archive_older'] != 0:
 854         repodirs.append('archive')
 855         if not os.path.exists('archive'):
 856             os.mkdir('archive')
 857
 858     if options.icons:
 859         resize_all_icons(repodirs)
 860         sys.exit(0)
 861
 862     # Get all apps...
 863     apps = metadata.read_metadata()
 864
 865     # Generate a list of categories...
 866     categories = []
 867     for app in apps:
 868         cats = app['Categories'].split(',')
 869         for cat in cats:
 870             if cat not in categories:
 871                 categories.append(cat)
 872
 873     # Read known apks data (will be updated and written back when we've finished)
 874     knownapks = common.KnownApks()
 875
 876     # Gather information about all the apk files in the repo directory, using
 877     # cached data if possible.
 878     apkcachefile = os.path.join('tmp', 'apkcache')
 879     if not options.clean and os.path.exists(apkcachefile):
 880         with open(apkcachefile, 'rb') as cf:
 881             apkcache = pickle.load(cf)
 882     else:
 883         apkcache = {}
 884     cachechanged = False
 885
 886     delete_disabled_builds(apps, apkcache, repodirs)
 887
 888     # Scan all apks in the main repo
 889     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 890     if cc:
 891         cachechanged = True
 892
 893     # Scan the archive repo for apks as well
 894     if len(repodirs) > 1:
 895         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 896         if cc:
 897             cachechanged = True
 898     else:
 899         archapks = []
 900
 901     # Some information from the apks needs to be applied up to the application
 902     # level. When doing this, we use the info from the most recent version's apk.
 903     # We deal with figuring out when the app was added and last updated at the
 904     # same time.
 905     for app in apps:
 906         bestver = 0
 907         added = None
 908         lastupdated = None
 909         for apk in apks + archapks:
 910             if apk['id'] == app['id']:
 911                 if apk['versioncode'] > bestver:
 912                     bestver = apk['versioncode']
 913                     bestapk = apk
 914
 915                 if 'added' in apk:
 916                     if not added or apk['added'] < added:
 917                         added = apk['added']
 918                     if not lastupdated or apk['added'] > lastupdated:
 919                         lastupdated = apk['added']
 920
 921         if added:
 922             app['added'] = added
 923         else:
 924             if options.verbose:
 925                 print "WARNING: Don't know when " + app['id'] + " was added"
 926         if lastupdated:
 927             app['lastupdated'] = lastupdated
 928         else:
 929             if options.verbose:
 930                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 931
 932         if bestver == 0:
 933             if app['Name'] is None:
 934                 app['Name'] = app['id']
 935             app['icon'] = None
 936             if options.verbose and app['Disabled'] is None:
 937                 print "WARNING: Application " + app['id'] + " has no packages"
 938         else:
 939             if app['Name'] is None:
 940                 app['Name'] = bestapk['name']
 941             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 942
 943     # Sort the app list by name, then the web site doesn't have to by default.
 944     # (we had to wait until we'd scanned the apks to do this, because mostly the
 945     # name comes from there!)
 946     apps = sorted(apps, key=lambda app: app['Name'].upper())
 947
 948     # Generate warnings for apk's with no metadata (or create skeleton
 949     # metadata files, if requested on the command line)
 950     for apk in apks:
 951         found = False
 952         for app in apps:
 953             if app['id'] == apk['id']:
 954                 found = True
 955                 break
 956         if not found:
 957             if options.createmeta:
 958                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 959                 f.write("License:Unknown\n")
 960                 f.write("Web Site:\n")
 961                 f.write("Source Code:\n")
 962                 f.write("Issue Tracker:\n")
 963                 f.write("Summary:" + apk['name'] + "\n")
 964                 f.write("Description:\n")
 965                 f.write(apk['name'] + "\n")
 966                 f.write(".\n")
 967                 f.close()
 968                 print "Generated skeleton metadata for " + apk['id']
 969             else:
 970                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 971                 print "       " + apk['name'] + " - " + apk['version']
 972
 973     if len(repodirs) > 1:
 974         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 975
 976     # Make the index for the main repo...
 977     make_index(apps, apks, repodirs[0], False, categories)
 978
 979     # If there's an archive repo,  make the index for it. We already scanned it
 980     # earlier on.
 981     if len(repodirs) > 1:
 982         make_index(apps, archapks, repodirs[1], True, categories)
 983
 984     if config['update_stats']:
 985
 986         # Update known apks info...
 987         knownapks.writeifchanged()
 988
 989         # Generate latest apps data for widget
 990         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 991             data = ''
 992             for line in file(os.path.join('stats', 'latestapps.txt')):
 993                 appid = line.rstrip()
 994                 data += appid + "\t"
 995                 for app in apps:
 996                     if app['id'] == appid:
 997                         data += app['Name'] + "\t"
 998                         if app['icon'] is not None:
 999                             data += app['icon'] + "\t"
1000                         data += app['License'] + "\n"
1001                         break
1002             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1003             f.write(data)
1004             f.close()
1005
1006     if cachechanged:
1007         with open(apkcachefile, 'wb') as cf:
1008             pickle.dump(apkcache, cf)
1009
1010     # Update the wiki...
1011     if options.wiki:
1012         update_wiki(apps, apks + archapks)
1013
1014     print "Finished."
1015
1016 if __name__ == "__main__":
1017     main()
1018