fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def launcher_size(density):
  42     return (int(density) * 48) / 160
  43
  44 def get_icon_dir(repodir, density):
  45     if density is None:
  46         return os.path.join(repodir, "icons")
  47     return os.path.join(repodir, "icons-%s" % density)
  48
  49 def get_icon_dirs(repodir):
  50     for density in get_densities():
  51         yield get_icon_dir(repodir, density)
  52     yield os.path.join(repodir, "icons")
  53
  54 def update_wiki(apps, apks):
  55     """Update the wiki
  56
  57     :param apps: fully populated list of all applications
  58     :param apks: all apks, except...
  59     """
  60     print "Updating wiki"
  61     wikicat = 'Apps'
  62     wikiredircat = 'App Redirects'
  63     import mwclient
  64     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  65             path=config['wiki_path'])
  66     site.login(config['wiki_user'], config['wiki_password'])
  67     generated_pages = {}
  68     generated_redirects = {}
  69     for app in apps:
  70         wikidata = ''
  71         if app['Disabled']:
  72             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  73         if app['AntiFeatures']:
  74             for af in app['AntiFeatures'].split(','):
  75                 wikidata += '{{AntiFeature|' + af + '}}\n'
  76         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  77                 app['id'],
  78                 app['Name'],
  79                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  80                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  81                 app['Source Code'],
  82                 app['Issue Tracker'],
  83                 app['Web Site'],
  84                 app['Donate'],
  85                 app['FlattrID'],
  86                 app['Bitcoin'],
  87                 app['Litecoin'],
  88                 app['Dogecoin'],
  89                 app['License'],
  90                 app.get('Requires Root', 'No'))
  91
  92         if app['Provides']:
  93             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  94
  95         wikidata += app['Summary']
  96         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  97
  98         wikidata += "=Description=\n"
  99         wikidata += metadata.description_wiki(app['Description']) + "\n"
 100
 101         wikidata += "=Maintainer Notes=\n"
 102         if 'Maintainer Notes' in app:
 103             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 104         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 105
 106         # Get a list of all packages for this application...
 107         apklist = []
 108         gotcurrentver = False
 109         cantupdate = False
 110         buildfails = False
 111         for apk in apks:
 112             if apk['id'] == app['id']:
 113                 if str(apk['versioncode']) == app['Current Version Code']:
 114                     gotcurrentver = True
 115                 apklist.append(apk)
 116         # Include ones we can't build, as a special case...
 117         for thisbuild in app['builds']:
 118             if 'disable' in thisbuild:
 119                 if thisbuild['vercode'] == app['Current Version Code']:
 120                     cantupdate = True
 121                 apklist.append({
 122                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 123                         'versioncode': int(thisbuild['vercode']),
 124                         'version': thisbuild['version'],
 125                         'buildproblem': thisbuild['disable']
 126                     })
 127             else:
 128                 builtit = False
 129                 for apk in apklist:
 130                     if apk['versioncode'] == int(thisbuild['vercode']):
 131                         builtit = True
 132                         break
 133                 if not builtit:
 134                     buildfails = True
 135                     apklist.append({
 136                             'versioncode': int(thisbuild['vercode']),
 137                             'version': thisbuild['version'],
 138                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 139                         })
 140         # Sort with most recent first...
 141         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 142
 143         wikidata += "=Versions=\n"
 144         if len(apklist) == 0:
 145             wikidata += "We currently have no versions of this app available."
 146         elif not gotcurrentver:
 147             wikidata += "We don't have the current version of this app."
 148         else:
 149             wikidata += "We have the current version of this app."
 150         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 151         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 152         if len(app['No Source Since']) > 0:
 153             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 154         if len(app['Current Version']) > 0:
 155             wikidata += "The current (recommended) version is " + app['Current Version']
 156             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 157         validapks = 0
 158         for apk in apklist:
 159             wikidata += "==" + apk['version'] + "==\n"
 160
 161             if 'buildproblem' in apk:
 162                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 163             else:
 164                 validapks += 1
 165                 wikidata += "This version is built and signed by "
 166                 if 'srcname' in apk:
 167                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 168                 else:
 169                     wikidata += "the original developer.\n\n"
 170             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 171
 172         wikidata += '\n[[Category:' + wikicat + ']]\n'
 173         if len(app['No Source Since']) > 0:
 174             wikidata += '\n[[Category:Apps missing source code]]\n'
 175         elif validapks == 0 and not app['Disabled']:
 176             wikidata += '\n[[Category:Apps with no packages]]\n'
 177         elif cantupdate and not app['Disabled']:
 178             wikidata += "\n[[Category:Apps we can't update]]\n"
 179         elif buildfails and not app['Disabled']:
 180             wikidata += "\n[[Category:Apps with failing builds]]\n"
 181         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 182             wikidata += '\n[[Category:Apps to Update]]\n'
 183         if app['Disabled']:
 184             wikidata += '\n[[Category:Apps that are disabled]]\n'
 185         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 186             wikidata += '\n[[Category:Apps with no update check]]\n'
 187         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 188             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 189
 190         # We can't have underscores in the page name, even if they're in
 191         # the package ID, because MediaWiki messes with them...
 192         pagename = app['id'].replace('_', ' ')
 193
 194         # Drop a trailing newline, because mediawiki is going to drop it anyway
 195         # and it we don't we'll think the page has changed when it hasn't...
 196         if wikidata.endswith('\n'):
 197             wikidata = wikidata[:-1]
 198
 199         generated_pages[pagename] = wikidata
 200
 201         # Make a redirect from the name to the ID too, unless there's
 202         # already an existing page with the name and it isn't a redirect.
 203         noclobber = False
 204         apppagename = app['Name'].replace('_', ' ')
 205         apppagename = apppagename.replace('{', '')
 206         apppagename = apppagename.replace('}', ' ')
 207         apppagename = apppagename.replace(':', ' ')
 208         # Drop double spaces caused mostly by replacing ':' above
 209         apppagename = apppagename.replace('  ', ' ')
 210         for expagename in site.allpages(prefix=apppagename,
 211                 filterredir='nonredirects', generator=False):
 212             if expagename == apppagename:
 213                 noclobber = True
 214         # Another reason not to make the redirect page is if the app name
 215         # is the same as it's ID, because that will overwrite the real page
 216         # with an redirect to itself! (Although it seems like an odd
 217         # scenario this happens a lot, e.g. where there is metadata but no
 218         # builds or binaries to extract a name from.
 219         if apppagename == pagename:
 220             noclobber = True
 221         if not noclobber:
 222             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 223
 224     for tcat, genp in [(wikicat, generated_pages),
 225             (wikiredircat, generated_redirects)]:
 226         catpages = site.Pages['Category:' + tcat]
 227         existingpages = []
 228         for page in catpages:
 229             existingpages.append(page.name)
 230             if page.name in genp:
 231                 pagetxt = page.edit()
 232                 if pagetxt != genp[page.name]:
 233                     print "Updating modified page " + page.name
 234                     page.save(genp[page.name], summary='Auto-updated')
 235                 else:
 236                     if options.verbose:
 237                         print "Page " + page.name + " is unchanged"
 238             else:
 239                 print "Deleting page " + page.name
 240                 page.delete('No longer published')
 241         for pagename, text in genp.items():
 242             if options.verbose:
 243                 print "Checking " + pagename
 244             if not pagename in existingpages:
 245                 print "Creating page " + pagename
 246                 try:
 247                     newpage = site.Pages[pagename]
 248                     newpage.save(text, summary='Auto-created')
 249                 except:
 250                     print "...FAILED to create page"
 251
 252     # Purge server cache to ensure counts are up to date
 253     site.pages['Repository Maintenance'].purge()
 254
 255 def delete_disabled_builds(apps, apkcache, repodirs):
 256     """Delete disabled build outputs.
 257
 258     :param apps: list of all applications, as per metadata.read_metadata
 259     :param apkcache: current apk cache information
 260     :param repodirs: the repo directories to process
 261     """
 262     for app in apps:
 263         for build in app['builds']:
 264             if 'disable' in build:
 265                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 266                 for repodir in repodirs:
 267                     apkpath = os.path.join(repodir, apkfilename)
 268                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 269                     for name in [apkpath, srcpath]:
 270                         if os.path.exists(name):
 271                             print "Deleting disabled build output " + apkfilename
 272                             os.remove(name)
 273                 if apkfilename in apkcache:
 274                     del apkcache[apkfilename]
 275
 276 def resize_icon(iconpath, density):
 277
 278     try:
 279         im = Image.open(iconpath)
 280         size = launcher_size(density)
 281
 282         if any(length > size for length in im.size):
 283             oldsize = im.size
 284             im.thumbnail((size, size), Image.ANTIALIAS)
 285             print iconpath, "was too large at", oldsize, "- new size is", im.size
 286             im.save(iconpath, "PNG")
 287
 288         else:
 289             if options.verbose:
 290                 print iconpath, "is small enough:", im.size
 291
 292     except Exception,e:
 293         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 294
 295 def resize_all_icons(repodirs):
 296     """Resize all icons that exceed the max size
 297
 298     :param repodirs: the repo directories to process
 299     """
 300     for repodir in repodirs:
 301         for density in get_densities():
 302             icon_dir = get_icon_dir(repodir, density)
 303             icon_glob = os.path.join(icon_dir, '*.png')
 304             for iconpath in glob.glob(icon_glob):
 305                 resize_icon(iconpath, density)
 306
 307 def scan_apks(apps, apkcache, repodir, knownapks):
 308     """Scan the apks in the given repo directory.
 309
 310     This also extracts the icons.
 311
 312     :param apps: list of all applications, as per metadata.read_metadata
 313     :param apkcache: current apk cache information
 314     :param repodir: repo directory to scan
 315     :param knownapks: known apks info
 316     :returns: (apks, cachechanged) where apks is a list of apk information,
 317               and cachechanged is True if the apkcache got changed.
 318     """
 319
 320     cachechanged = False
 321
 322     icon_dirs = get_icon_dirs(repodir)
 323     for icon_dir in icon_dirs:
 324         if os.path.exists(icon_dir):
 325             if options.clean:
 326                 shutil.rmtree(icon_dir)
 327                 os.makedirs(icon_dir)
 328         else:
 329             os.makedirs(icon_dir)
 330
 331     apks = []
 332     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 333     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 334     vername_pat = re.compile(".*versionName='([^']*)'.*")
 335     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 336     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 337     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 338     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 339     string_pat = re.compile(".*'([^']*)'.*")
 340     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 341
 342         apkfilename = apkfile[len(repodir) + 1:]
 343         if ' ' in apkfilename:
 344             print "No spaces in APK filenames!"
 345             sys.exit(1)
 346
 347         if apkfilename in apkcache:
 348             if options.verbose:
 349                 print "Reading " + apkfilename + " from cache"
 350             thisinfo = apkcache[apkfilename]
 351
 352         else:
 353
 354             if not options.quiet:
 355                 print "Processing " + apkfilename
 356             thisinfo = {}
 357             thisinfo['apkname'] = apkfilename
 358             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 359             if os.path.exists(os.path.join(repodir, srcfilename)):
 360                 thisinfo['srcname'] = srcfilename
 361             thisinfo['size'] = os.path.getsize(apkfile)
 362             thisinfo['permissions'] = []
 363             thisinfo['features'] = []
 364             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 365                                   'dump', 'badging', apkfile],
 366                                  stdout=subprocess.PIPE)
 367             output = p.communicate()[0]
 368             if p.returncode != 0:
 369                 print "ERROR: Failed to get apk information"
 370                 sys.exit(1)
 371             for line in output.splitlines():
 372                 if line.startswith("package:"):
 373                     try:
 374                         thisinfo['id'] = re.match(name_pat, line).group(1)
 375                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 376                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 377                     except Exception, e:
 378                         print "Package matching failed: " + str(e)
 379                         print "Line was: " + line
 380                         sys.exit(1)
 381                 elif line.startswith("application:"):
 382                     thisinfo['name'] = re.match(label_pat, line).group(1)
 383                     if 'icons_src' not in thisinfo:
 384                         thisinfo['icons_src'] = {}
 385                     # Keep path to non-dpi icon in case we need it
 386                     match = re.match(icon_pat_nodpi, line)
 387                     if match:
 388                         thisinfo['icons_src']['-1'] = match.group(1)
 389                 elif line.startswith("launchable-activity:"):
 390                     if 'icons_src' not in thisinfo:
 391                         thisinfo['icons_src'] = {}
 392                     # Only use launchable-activity as fallback to application
 393                     elif '-1' not in thisinfo['icons_src']:
 394                         match = re.match(icon_pat_nodpi, line)
 395                         if match:
 396                             thisinfo['icons_src']['-1'] = match.group(1)
 397                 elif line.startswith("application-icon-"):
 398                     if 'icons_src' not in thisinfo:
 399                         thisinfo['icons_src'] = {}
 400                     match = re.match(icon_pat, line)
 401                     if match:
 402                         density = match.group(1)
 403                         path = match.group(2)
 404                         thisinfo['icons_src'][density] = path
 405                 elif line.startswith("sdkVersion:"):
 406                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 407                 elif line.startswith("native-code:"):
 408                     thisinfo['nativecode'] = []
 409                     for arch in line[13:].split(' '):
 410                         thisinfo['nativecode'].append(arch[1:-1])
 411                 elif line.startswith("uses-permission:"):
 412                     perm = re.match(string_pat, line).group(1)
 413                     if perm.startswith("android.permission."):
 414                         perm = perm[19:]
 415                     thisinfo['permissions'].append(perm)
 416                 elif line.startswith("uses-feature:"):
 417                     perm = re.match(string_pat, line).group(1)
 418                     #Filter out this, it's only added with the latest SDK tools and
 419                     #causes problems for lots of apps.
 420                     if (perm != "android.hardware.screen.portrait" and
 421                         perm != "android.hardware.screen.landscape"):
 422                         if perm.startswith("android.feature."):
 423                             perm = perm[16:]
 424                         thisinfo['features'].append(perm)
 425
 426             if not 'sdkversion' in thisinfo:
 427                 print "  WARNING: no SDK version information found"
 428                 thisinfo['sdkversion'] = 0
 429
 430             # Check for debuggable apks...
 431             if common.isApkDebuggable(apkfile, config):
 432                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 433
 434             # Calculate the sha256...
 435             sha = hashlib.sha256()
 436             with open(apkfile, 'rb') as f:
 437                 while True:
 438                     t = f.read(1024)
 439                     if len(t) == 0:
 440                         break
 441                     sha.update(t)
 442                 thisinfo['sha256'] = sha.hexdigest()
 443
 444             # Get the signature (or md5 of, to be precise)...
 445             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 446             if not os.path.exists(getsig_dir + "/getsig.class"):
 447                 print "ERROR: getsig.class not found. To fix:"
 448                 print "\tcd " + getsig_dir
 449                 print "\t./make.sh"
 450                 sys.exit(1)
 451             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 452                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 453             output = p.communicate()[0]
 454             if options.verbose:
 455                 print output
 456             if p.returncode != 0 or not output.startswith('Result:'):
 457                 print "ERROR: Failed to get apk signature"
 458                 sys.exit(1)
 459             thisinfo['sig'] = output[7:].strip()
 460
 461             apk = zipfile.ZipFile(apkfile, 'r')
 462
 463             iconfilename = "%s.%s.png" % (
 464                     thisinfo['id'],
 465                     thisinfo['versioncode'])
 466
 467             # Extract the icon file...
 468             densities = get_densities()
 469             empty_densities = []
 470             for density in densities:
 471                 if density not in thisinfo['icons_src']:
 472                     empty_densities.append(density)
 473                     continue
 474                 if 'icons' not in thisinfo:
 475                     thisinfo['icons'] = {}
 476                 iconsrc = thisinfo['icons_src'][density]
 477                 icon_dir = get_icon_dir(repodir, density)
 478                 icondest = os.path.join(icon_dir, iconfilename)
 479
 480                 try:
 481                     iconfile = open(icondest, 'wb')
 482                     iconfile.write(apk.read(iconsrc))
 483                     iconfile.close()
 484                     thisinfo['icons'][density] = iconfilename
 485
 486                 except:
 487                     print "WARNING: Error retrieving icon file"
 488                     del thisinfo['icons'][density]
 489                     del thisinfo['icons_src'][density]
 490                     empty_densities.append(density)
 491
 492                 resize_icon(icondest, density)
 493
 494             apk.close()
 495
 496             # First try resizing down to not lose quality
 497             last_density = None
 498             for density in densities:
 499                 if density not in empty_densities:
 500                     last_density = density
 501                     continue
 502                 if last_density is None:
 503                     continue
 504                 if options.verbose:
 505                     print "Density %s not available, resizing down from %s" % (
 506                             density, last_density)
 507
 508                 last_iconpath = os.path.join(
 509                         get_icon_dir(repodir, last_density), iconfilename)
 510                 iconpath = os.path.join(
 511                         get_icon_dir(repodir, density), iconfilename)
 512                 im = Image.open(last_iconpath)
 513                 size = launcher_size(density)
 514
 515                 im.thumbnail((size, size), Image.ANTIALIAS)
 516                 im.save(iconpath, "PNG")
 517                 empty_densities.remove(density)
 518
 519             # Then just copy from the highest resolution available
 520             last_density = None
 521             for density in reversed(densities):
 522                 if density not in empty_densities:
 523                     last_density = density
 524                     continue
 525                 if last_density is None:
 526                     continue
 527                 if options.verbose:
 528                     print "Density %s not available, copying from lower density %s" % (
 529                             density, last_density)
 530
 531                 shutil.copyfile(
 532                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 533                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 534
 535                 empty_densities.remove(density)
 536
 537             # Copy from icons-mdpi to icons since mdpi is the baseline density
 538             shutil.copyfile(
 539                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 540                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 541
 542             # Record in known apks, getting the added date at the same time..
 543             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 544             if added:
 545                 thisinfo['added'] = added
 546
 547             apkcache[apkfilename] = thisinfo
 548             cachechanged = True
 549
 550         apks.append(thisinfo)
 551
 552     return apks, cachechanged
 553
 554
 555 repo_pubkey_fingerprint = None
 556
 557 def make_index(apps, apks, repodir, archive, categories):
 558     """Make a repo index.
 559
 560     :param apps: fully populated apps list
 561     :param apks: full populated apks list
 562     :param repodir: the repo directory
 563     :param archive: True if this is the archive repo, False if it's the
 564                     main one.
 565     :param categories: list of categories
 566     """
 567
 568     doc = Document()
 569
 570     def addElement(name, value, doc, parent):
 571         el = doc.createElement(name)
 572         el.appendChild(doc.createTextNode(value))
 573         parent.appendChild(el)
 574     def addElementCDATA(name, value, doc, parent):
 575         el = doc.createElement(name)
 576         el.appendChild(doc.createCDATASection(value))
 577         parent.appendChild(el)
 578
 579     root = doc.createElement("fdroid")
 580     doc.appendChild(root)
 581
 582     repoel = doc.createElement("repo")
 583
 584     if archive:
 585         repoel.setAttribute("name", config['archive_name'])
 586         if config['repo_maxage'] != 0:
 587             repoel.setAttribute("maxage", str(config['repo_maxage']))
 588         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 589         repoel.setAttribute("url", config['archive_url'])
 590         addElement('description', config['archive_description'], doc, repoel)
 591
 592     else:
 593         repoel.setAttribute("name", config['repo_name'])
 594         if config['repo_maxage'] != 0:
 595             repoel.setAttribute("maxage", str(config['repo_maxage']))
 596         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 597         repoel.setAttribute("url", config['repo_url'])
 598         addElement('description', config['repo_description'], doc, repoel)
 599
 600     repoel.setAttribute("version", "10")
 601     repoel.setAttribute("timestamp", str(int(time.time())))
 602
 603     if config['repo_keyalias']:
 604
 605         # Generate a certificate fingerprint the same way keytool does it
 606         # (but with slightly different formatting)
 607         def cert_fingerprint(data):
 608             digest = hashlib.sha1(data).digest()
 609             ret = []
 610             for i in range(4):
 611                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 612             return " ".join(ret)
 613
 614         def extract_pubkey():
 615             p = subprocess.Popen(['keytool', '-exportcert',
 616                                   '-alias', config['repo_keyalias'],
 617                                   '-keystore', config['keystore'],
 618                                   '-storepass', config['keystorepass']],
 619                                  stdout=subprocess.PIPE)
 620             cert = p.communicate()[0]
 621             if p.returncode != 0:
 622                 print "ERROR: Failed to get repo pubkey"
 623                 sys.exit(1)
 624             global repo_pubkey_fingerprint
 625             repo_pubkey_fingerprint = cert_fingerprint(cert)
 626             return "".join("%02x" % ord(b) for b in cert)
 627
 628         repoel.setAttribute("pubkey", extract_pubkey())
 629
 630     root.appendChild(repoel)
 631
 632     for app in apps:
 633
 634         if app['Disabled'] is not None:
 635             continue
 636
 637         # Get a list of the apks for this app...
 638         apklist = []
 639         for apk in apks:
 640             if apk['id'] == app['id']:
 641                 apklist.append(apk)
 642
 643         if len(apklist) == 0:
 644             continue
 645
 646         apel = doc.createElement("application")
 647         apel.setAttribute("id", app['id'])
 648         root.appendChild(apel)
 649
 650         addElement('id', app['id'], doc, apel)
 651         if 'added' in app:
 652             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 653         if 'lastupdated' in app:
 654             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 655         addElement('name', app['Name'], doc, apel)
 656         addElement('summary', app['Summary'], doc, apel)
 657         if app['icon']:
 658             addElement('icon', app['icon'], doc, apel)
 659         def linkres(link):
 660             for app in apps:
 661                 if app['id'] == link:
 662                     return ("fdroid.app:" + link, app['Name'])
 663             raise MetaDataException("Cannot resolve app id " + link)
 664         addElement('desc',
 665                 metadata.description_html(app['Description'], linkres), doc, apel)
 666         addElement('license', app['License'], doc, apel)
 667         if 'Categories' in app:
 668             appcategories = [c.strip() for c in app['Categories'].split(',')]
 669             addElement('categories', ','.join(appcategories), doc, apel)
 670             # We put the first (primary) category in LAST, which will have
 671             # the desired effect of making clients that only understand one
 672             # category see that one.
 673             addElement('category', appcategories[0], doc, apel)
 674         addElement('web', app['Web Site'], doc, apel)
 675         addElement('source', app['Source Code'], doc, apel)
 676         addElement('tracker', app['Issue Tracker'], doc, apel)
 677         if app['Donate']:
 678             addElement('donate', app['Donate'], doc, apel)
 679         if app['Bitcoin']:
 680             addElement('bitcoin', app['Bitcoin'], doc, apel)
 681         if app['Litecoin']:
 682             addElement('litecoin', app['Litecoin'], doc, apel)
 683         if app['Dogecoin']:
 684             addElement('dogecoin', app['Dogecoin'], doc, apel)
 685         if app['FlattrID']:
 686             addElement('flattr', app['FlattrID'], doc, apel)
 687
 688         # These elements actually refer to the current version (i.e. which
 689         # one is recommended. They are historically mis-named, and need
 690         # changing, but stay like this for now to support existing clients.
 691         addElement('marketversion', app['Current Version'], doc, apel)
 692         addElement('marketvercode', app['Current Version Code'], doc, apel)
 693
 694         if app['AntiFeatures']:
 695             af = app['AntiFeatures'].split(',')
 696             # TODO: Temporarily not including UpstreamNonFree in the index,
 697             # because current F-Droid clients do not understand it, and also
 698             # look ugly when they encounter an unknown antifeature. This
 699             # filtering can be removed in time...
 700             if 'UpstreamNonFree' in af:
 701                 af.remove('UpstreamNonFree')
 702             if af:
 703                 addElement('antifeatures', ','.join(af), doc, apel)
 704         if app['Provides']:
 705             pv = app['Provides'].split(',')
 706             addElement('provides', ','.join(pv), doc, apel)
 707         if app['Requires Root']:
 708             addElement('requirements', 'root', doc, apel)
 709
 710         # Sort the apk list into version order, just so the web site
 711         # doesn't have to do any work by default...
 712         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 713
 714         # Check for duplicates - they will make the client unhappy...
 715         for i in range(len(apklist) - 1):
 716             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 717                 print "ERROR - duplicate versions"
 718                 print apklist[i]['apkname']
 719                 print apklist[i+1]['apkname']
 720                 sys.exit(1)
 721
 722         for apk in apklist:
 723             apkel = doc.createElement("package")
 724             apel.appendChild(apkel)
 725             addElement('version', apk['version'], doc, apkel)
 726             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 727             addElement('apkname', apk['apkname'], doc, apkel)
 728             if 'srcname' in apk:
 729                 addElement('srcname', apk['srcname'], doc, apkel)
 730             for hash_type in ['sha256']:
 731                 if not hash_type in apk:
 732                     continue
 733                 hashel = doc.createElement("hash")
 734                 hashel.setAttribute("type", hash_type)
 735                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 736                 apkel.appendChild(hashel)
 737             addElement('sig', apk['sig'], doc, apkel)
 738             addElement('size', str(apk['size']), doc, apkel)
 739             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 740             if 'added' in apk:
 741                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 742             if app['Requires Root']:
 743                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 744                     apk['permissions'].append('ACCESS_SUPERUSER')
 745
 746             if len(apk['permissions']) > 0:
 747                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 748             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 749                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 750             if len(apk['features']) > 0:
 751                 addElement('features', ','.join(apk['features']), doc, apkel)
 752
 753     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 754     if options.pretty:
 755         output = doc.toprettyxml()
 756     else:
 757         output = doc.toxml()
 758     of.write(output)
 759     of.close()
 760
 761     if config['repo_keyalias'] is not None:
 762
 763         if not options.quiet:
 764             print "Creating signed index."
 765             print "Key fingerprint:", repo_pubkey_fingerprint
 766
 767         #Create a jar of the index...
 768         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 769             cwd=repodir, stdout=subprocess.PIPE)
 770         output = p.communicate()[0]
 771         if options.verbose:
 772             print output
 773         if p.returncode != 0:
 774             print "ERROR: Failed to create jar file"
 775             sys.exit(1)
 776
 777         # Sign the index...
 778         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 779             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 780             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 781             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 782         output = p.communicate()[0]
 783         if p.returncode != 0:
 784             print "Failed to sign index"
 785             print output
 786             sys.exit(1)
 787         if options.verbose:
 788             print output
 789
 790     # Copy the repo icon into the repo directory...
 791     icon_dir = os.path.join(repodir ,'icons')
 792     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 793     shutil.copyfile(config['repo_icon'], iconfilename)
 794
 795     # Write a category list in the repo to allow quick access...
 796     catdata = ''
 797     for cat in categories:
 798         catdata += cat + '\n'
 799     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 800     f.write(catdata)
 801     f.close()
 802
 803
 804
 805 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 806
 807     for app in apps:
 808
 809         # Get a list of the apks for this app...
 810         apklist = []
 811         for apk in apks:
 812             if apk['id'] == app['id']:
 813                 apklist.append(apk)
 814
 815         # Sort the apk list into version order...
 816         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 817
 818         if app['Archive Policy']:
 819             keepversions = int(app['Archive Policy'][:-9])
 820         else:
 821             keepversions = defaultkeepversions
 822
 823         if len(apklist) > keepversions:
 824             for apk in apklist[keepversions:]:
 825                 print "Moving " + apk['apkname'] + " to archive"
 826                 shutil.move(os.path.join(repodir, apk['apkname']),
 827                     os.path.join(archivedir, apk['apkname']))
 828                 if 'srcname' in apk:
 829                     shutil.move(os.path.join(repodir, apk['srcname']),
 830                         os.path.join(archivedir, apk['srcname']))
 831                 apks.remove(apk)
 832
 833
 834 config = None
 835 options = None
 836
 837 def main():
 838
 839     global config, options
 840
 841     # Parse command line...
 842     parser = OptionParser()
 843     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 844                       help="Create skeleton metadata files that are missing")
 845     parser.add_option("-v", "--verbose", action="store_true", default=False,
 846                       help="Spew out even more information than normal")
 847     parser.add_option("-q", "--quiet", action="store_true", default=False,
 848                       help="No output, except for warnings and errors")
 849     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 850                       help="Report on build data status")
 851     parser.add_option("-i", "--interactive", default=False, action="store_true",
 852                       help="Interactively ask about things that need updating.")
 853     parser.add_option("-I", "--icons", action="store_true", default=False,
 854                       help="Resize all the icons exceeding the max pixel size and exit")
 855     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 856                       help="Specify editor to use in interactive mode. Default "+
 857                           "is /etc/alternatives/editor")
 858     parser.add_option("-w", "--wiki", default=False, action="store_true",
 859                       help="Update the wiki")
 860     parser.add_option("", "--pretty", action="store_true", default=False,
 861                       help="Produce human-readable index.xml")
 862     parser.add_option("--clean", action="store_true", default=False,
 863                       help="Clean update - don't uses caches, reprocess all apks")
 864     (options, args) = parser.parse_args()
 865
 866     config = common.read_config(options)
 867
 868     repodirs = ['repo']
 869     if config['archive_older'] != 0:
 870         repodirs.append('archive')
 871         if not os.path.exists('archive'):
 872             os.mkdir('archive')
 873
 874     if options.icons:
 875         resize_all_icons(repodirs)
 876         sys.exit(0)
 877
 878     # Get all apps...
 879     apps = metadata.read_metadata()
 880
 881     # Generate a list of categories...
 882     categories = []
 883     for app in apps:
 884         cats = app['Categories'].split(',')
 885         for cat in cats:
 886             if cat not in categories:
 887                 categories.append(cat)
 888
 889     # Read known apks data (will be updated and written back when we've finished)
 890     knownapks = common.KnownApks()
 891
 892     # Gather information about all the apk files in the repo directory, using
 893     # cached data if possible.
 894     apkcachefile = os.path.join('tmp', 'apkcache')
 895     if not options.clean and os.path.exists(apkcachefile):
 896         with open(apkcachefile, 'rb') as cf:
 897             apkcache = pickle.load(cf)
 898     else:
 899         apkcache = {}
 900     cachechanged = False
 901
 902     delete_disabled_builds(apps, apkcache, repodirs)
 903
 904     # Scan all apks in the main repo
 905     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 906     if cc:
 907         cachechanged = True
 908
 909     # Scan the archive repo for apks as well
 910     if len(repodirs) > 1:
 911         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 912         if cc:
 913             cachechanged = True
 914     else:
 915         archapks = []
 916
 917     # Some information from the apks needs to be applied up to the application
 918     # level. When doing this, we use the info from the most recent version's apk.
 919     # We deal with figuring out when the app was added and last updated at the
 920     # same time.
 921     for app in apps:
 922         bestver = 0
 923         added = None
 924         lastupdated = None
 925         for apk in apks + archapks:
 926             if apk['id'] == app['id']:
 927                 if apk['versioncode'] > bestver:
 928                     bestver = apk['versioncode']
 929                     bestapk = apk
 930
 931                 if 'added' in apk:
 932                     if not added or apk['added'] < added:
 933                         added = apk['added']
 934                     if not lastupdated or apk['added'] > lastupdated:
 935                         lastupdated = apk['added']
 936
 937         if added:
 938             app['added'] = added
 939         else:
 940             if options.verbose:
 941                 print "WARNING: Don't know when " + app['id'] + " was added"
 942         if lastupdated:
 943             app['lastupdated'] = lastupdated
 944         else:
 945             if options.verbose:
 946                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 947
 948         if bestver == 0:
 949             if app['Name'] is None:
 950                 app['Name'] = app['id']
 951             app['icon'] = None
 952             if options.verbose and app['Disabled'] is None:
 953                 print "WARNING: Application " + app['id'] + " has no packages"
 954         else:
 955             if app['Name'] is None:
 956                 app['Name'] = bestapk['name']
 957             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 958
 959     # Sort the app list by name, then the web site doesn't have to by default.
 960     # (we had to wait until we'd scanned the apks to do this, because mostly the
 961     # name comes from there!)
 962     apps = sorted(apps, key=lambda app: app['Name'].upper())
 963
 964     # Generate warnings for apk's with no metadata (or create skeleton
 965     # metadata files, if requested on the command line)
 966     for apk in apks:
 967         found = False
 968         for app in apps:
 969             if app['id'] == apk['id']:
 970                 found = True
 971                 break
 972         if not found:
 973             if options.createmeta:
 974                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 975                 f.write("License:Unknown\n")
 976                 f.write("Web Site:\n")
 977                 f.write("Source Code:\n")
 978                 f.write("Issue Tracker:\n")
 979                 f.write("Summary:" + apk['name'] + "\n")
 980                 f.write("Description:\n")
 981                 f.write(apk['name'] + "\n")
 982                 f.write(".\n")
 983                 f.close()
 984                 print "Generated skeleton metadata for " + apk['id']
 985             else:
 986                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 987                 print "       " + apk['name'] + " - " + apk['version']
 988
 989     if len(repodirs) > 1:
 990         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 991
 992     # Make the index for the main repo...
 993     make_index(apps, apks, repodirs[0], False, categories)
 994
 995     # If there's an archive repo,  make the index for it. We already scanned it
 996     # earlier on.
 997     if len(repodirs) > 1:
 998         make_index(apps, archapks, repodirs[1], True, categories)
 999
1000     if config['update_stats']:
1001
1002         # Update known apks info...
1003         knownapks.writeifchanged()
1004
1005         # Generate latest apps data for widget
1006         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1007             data = ''
1008             for line in file(os.path.join('stats', 'latestapps.txt')):
1009                 appid = line.rstrip()
1010                 data += appid + "\t"
1011                 for app in apps:
1012                     if app['id'] == appid:
1013                         data += app['Name'] + "\t"
1014                         if app['icon'] is not None:
1015                             data += app['icon'] + "\t"
1016                         data += app['License'] + "\n"
1017                         break
1018             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1019             f.write(data)
1020             f.close()
1021
1022     if cachechanged:
1023         with open(apkcachefile, 'wb') as cf:
1024             pickle.dump(apkcache, cf)
1025
1026     # Update the wiki...
1027     if options.wiki:
1028         update_wiki(apps, apks + archapks)
1029
1030     print "Finished."
1031
1032 if __name__ == "__main__":
1033     main()
1034