fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def launcher_size(density):
  42     return (int(density) * 48) / 160
  43
  44 def get_icon_dir(repodir, density):
  45     if density is None:
  46         return os.path.join(repodir, "icons")
  47     return os.path.join(repodir, "icons-%s" % density)
  48
  49 def get_icon_dirs(repodir):
  50     for density in get_densities():
  51         yield get_icon_dir(repodir, density)
  52     yield os.path.join(repodir, "icons")
  53
  54 def update_wiki(apps, apks):
  55     """Update the wiki
  56
  57     :param apps: fully populated list of all applications
  58     :param apks: all apks, except...
  59     """
  60     print "Updating wiki"
  61     wikicat = 'Apps'
  62     wikiredircat = 'App Redirects'
  63     import mwclient
  64     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  65             path=config['wiki_path'])
  66     site.login(config['wiki_user'], config['wiki_password'])
  67     generated_pages = {}
  68     generated_redirects = {}
  69     for app in apps:
  70         wikidata = ''
  71         if app['Disabled']:
  72             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  73         if app['AntiFeatures']:
  74             for af in app['AntiFeatures'].split(','):
  75                 wikidata += '{{AntiFeature|' + af + '}}\n'
  76         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  77                 app['id'],
  78                 app['Name'],
  79                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  80                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  81                 app['Source Code'],
  82                 app['Issue Tracker'],
  83                 app['Web Site'],
  84                 app['Donate'],
  85                 app['FlattrID'],
  86                 app['Bitcoin'],
  87                 app['Litecoin'],
  88                 app['Dogecoin'],
  89                 app['License'],
  90                 app.get('Requires Root', 'No'))
  91
  92         if app['Provides']:
  93             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  94
  95         wikidata += app['Summary']
  96         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  97
  98         wikidata += "=Description=\n"
  99         wikidata += metadata.description_wiki(app['Description']) + "\n"
 100
 101         wikidata += "=Maintainer Notes=\n"
 102         if 'Maintainer Notes' in app:
 103             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 104         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 105
 106         # Get a list of all packages for this application...
 107         apklist = []
 108         gotcurrentver = False
 109         cantupdate = False
 110         buildfails = False
 111         for apk in apks:
 112             if apk['id'] == app['id']:
 113                 if str(apk['versioncode']) == app['Current Version Code']:
 114                     gotcurrentver = True
 115                 apklist.append(apk)
 116         # Include ones we can't build, as a special case...
 117         for thisbuild in app['builds']:
 118             if 'disable' in thisbuild:
 119                 if thisbuild['vercode'] == app['Current Version Code']:
 120                     cantupdate = True
 121                 apklist.append({
 122                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 123                         'versioncode': int(thisbuild['vercode']),
 124                         'version': thisbuild['version'],
 125                         'buildproblem': thisbuild['disable']
 126                     })
 127             else:
 128                 builtit = False
 129                 for apk in apklist:
 130                     if apk['versioncode'] == int(thisbuild['vercode']):
 131                         builtit = True
 132                         break
 133                 if not builtit:
 134                     buildfails = True
 135                     apklist.append({
 136                             'versioncode': int(thisbuild['vercode']),
 137                             'version': thisbuild['version'],
 138                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 139                         })
 140         # Sort with most recent first...
 141         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 142
 143         wikidata += "=Versions=\n"
 144         if len(apklist) == 0:
 145             wikidata += "We currently have no versions of this app available."
 146         elif not gotcurrentver:
 147             wikidata += "We don't have the current version of this app."
 148         else:
 149             wikidata += "We have the current version of this app."
 150         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 151         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 152         if len(app['No Source Since']) > 0:
 153             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 154         if len(app['Current Version']) > 0:
 155             wikidata += "The current (recommended) version is " + app['Current Version']
 156             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 157         validapks = 0
 158         for apk in apklist:
 159             wikidata += "==" + apk['version'] + "==\n"
 160
 161             if 'buildproblem' in apk:
 162                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 163             else:
 164                 validapks += 1
 165                 wikidata += "This version is built and signed by "
 166                 if 'srcname' in apk:
 167                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 168                 else:
 169                     wikidata += "the original developer.\n\n"
 170             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 171
 172         wikidata += '\n[[Category:' + wikicat + ']]\n'
 173         if len(app['No Source Since']) > 0:
 174             wikidata += '\n[[Category:Apps missing source code]]\n'
 175         elif validapks == 0 and not app['Disabled']:
 176             wikidata += '\n[[Category:Apps with no packages]]\n'
 177         elif cantupdate and not app['Disabled']:
 178             wikidata += "\n[[Category:Apps we can't update]]\n"
 179         elif buildfails and not app['Disabled']:
 180             wikidata += "\n[[Category:Apps with failing builds]]\n"
 181         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 182             wikidata += '\n[[Category:Apps to Update]]\n'
 183         if app['Disabled']:
 184             wikidata += '\n[[Category:Apps that are disabled]]\n'
 185         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 186             wikidata += '\n[[Category:Apps with no update check]]\n'
 187         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 188             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 189
 190         # We can't have underscores in the page name, even if they're in
 191         # the package ID, because MediaWiki messes with them...
 192         pagename = app['id'].replace('_', ' ')
 193
 194         # Drop a trailing newline, because mediawiki is going to drop it anyway
 195         # and it we don't we'll think the page has changed when it hasn't...
 196         if wikidata.endswith('\n'):
 197             wikidata = wikidata[:-1]
 198
 199         generated_pages[pagename] = wikidata
 200
 201         # Make a redirect from the name to the ID too, unless there's
 202         # already an existing page with the name and it isn't a redirect.
 203         noclobber = False
 204         apppagename = app['Name'].replace('_', ' ')
 205         apppagename = apppagename.replace('{', '')
 206         apppagename = apppagename.replace('}', ' ')
 207         apppagename = apppagename.replace(':', ' ')
 208         # Drop double spaces caused mostly by replacing ':' above
 209         apppagename = apppagename.replace('  ', ' ')
 210         for expagename in site.allpages(prefix=apppagename,
 211                 filterredir='nonredirects', generator=False):
 212             if expagename == apppagename:
 213                 noclobber = True
 214         # Another reason not to make the redirect page is if the app name
 215         # is the same as it's ID, because that will overwrite the real page
 216         # with an redirect to itself! (Although it seems like an odd
 217         # scenario this happens a lot, e.g. where there is metadata but no
 218         # builds or binaries to extract a name from.
 219         if apppagename == pagename:
 220             noclobber = True
 221         if not noclobber:
 222             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 223
 224     for tcat, genp in [(wikicat, generated_pages),
 225             (wikiredircat, generated_redirects)]:
 226         catpages = site.Pages['Category:' + tcat]
 227         existingpages = []
 228         for page in catpages:
 229             existingpages.append(page.name)
 230             if page.name in genp:
 231                 pagetxt = page.edit()
 232                 if pagetxt != genp[page.name]:
 233                     print "Updating modified page " + page.name
 234                     page.save(genp[page.name], summary='Auto-updated')
 235                 else:
 236                     if options.verbose:
 237                         print "Page " + page.name + " is unchanged"
 238             else:
 239                 print "Deleting page " + page.name
 240                 page.delete('No longer published')
 241         for pagename, text in genp.items():
 242             if options.verbose:
 243                 print "Checking " + pagename
 244             if not pagename in existingpages:
 245                 print "Creating page " + pagename
 246                 try:
 247                     newpage = site.Pages[pagename]
 248                     newpage.save(text, summary='Auto-created')
 249                 except:
 250                     print "...FAILED to create page"
 251
 252     # Purge server cache to ensure counts are up to date
 253     site.pages['Repository Maintenance'].purge()
 254
 255 def delete_disabled_builds(apps, apkcache, repodirs):
 256     """Delete disabled build outputs.
 257
 258     :param apps: list of all applications, as per metadata.read_metadata
 259     :param apkcache: current apk cache information
 260     :param repodirs: the repo directories to process
 261     """
 262     for app in apps:
 263         for build in app['builds']:
 264             if 'disable' in build:
 265                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 266                 for repodir in repodirs:
 267                     apkpath = os.path.join(repodir, apkfilename)
 268                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 269                     for name in [apkpath, srcpath]:
 270                         if os.path.exists(name):
 271                             print "Deleting disabled build output " + apkfilename
 272                             os.remove(name)
 273                 if apkfilename in apkcache:
 274                     del apkcache[apkfilename]
 275
 276 def resize_icon(iconpath, density):
 277
 278     try:
 279         im = Image.open(iconpath)
 280         size = launcher_size(density)
 281
 282         if any(length > size for length in im.size):
 283             print iconpath, "is too large:", im.size
 284             im.thumbnail((size, size), Image.ANTIALIAS)
 285             print iconpath, "new size:", im.size
 286             im.save(iconpath, "PNG")
 287
 288         else:
 289             if options.verbose:
 290                 print iconpath, "is small enough:", im.size
 291
 292     except Exception,e:
 293         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 294
 295 def resize_all_icons(repodirs):
 296     """Resize all icons that exceed the max size
 297
 298     :param repodirs: the repo directories to process
 299     """
 300     for repodir in repodirs:
 301         for density in get_densities():
 302             icon_dir = get_icon_dir(repodir, density)
 303             icon_glob = os.path.join(icon_dir, '*.png')
 304             for iconpath in glob.glob(icon_glob):
 305                 resize_icon(iconpath, density)
 306
 307 def scan_apks(apps, apkcache, repodir, knownapks):
 308     """Scan the apks in the given repo directory.
 309
 310     This also extracts the icons.
 311
 312     :param apps: list of all applications, as per metadata.read_metadata
 313     :param apkcache: current apk cache information
 314     :param repodir: repo directory to scan
 315     :param knownapks: known apks info
 316     :returns: (apks, cachechanged) where apks is a list of apk information,
 317               and cachechanged is True if the apkcache got changed.
 318     """
 319
 320     cachechanged = False
 321
 322     icon_dirs = get_icon_dirs(repodir)
 323     for icon_dir in icon_dirs:
 324         if os.path.exists(icon_dir):
 325             if options.clean:
 326                 shutil.rmtree(icon_dir)
 327                 os.makedirs(icon_dir)
 328         else:
 329             os.makedirs(icon_dir)
 330
 331     apks = []
 332     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 333     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 334     vername_pat = re.compile(".*versionName='([^']*)'.*")
 335     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 336     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 337     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 338     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 339     string_pat = re.compile(".*'([^']*)'.*")
 340     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 341
 342         apkfilename = apkfile[len(repodir) + 1:]
 343         if ' ' in apkfilename:
 344             print "No spaces in APK filenames!"
 345             sys.exit(1)
 346
 347         if apkfilename in apkcache:
 348             if options.verbose:
 349                 print "Reading " + apkfilename + " from cache"
 350             thisinfo = apkcache[apkfilename]
 351
 352         else:
 353
 354             if not options.quiet:
 355                 print "Processing " + apkfilename
 356             thisinfo = {}
 357             thisinfo['apkname'] = apkfilename
 358             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 359             if os.path.exists(os.path.join(repodir, srcfilename)):
 360                 thisinfo['srcname'] = srcfilename
 361             thisinfo['size'] = os.path.getsize(apkfile)
 362             thisinfo['permissions'] = []
 363             thisinfo['features'] = []
 364             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 365                                   'dump', 'badging', apkfile],
 366                                  stdout=subprocess.PIPE)
 367             output = p.communicate()[0]
 368             if p.returncode != 0:
 369                 print "ERROR: Failed to get apk information"
 370                 sys.exit(1)
 371             for line in output.splitlines():
 372                 if line.startswith("package:"):
 373                     try:
 374                         thisinfo['id'] = re.match(name_pat, line).group(1)
 375                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 376                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 377                     except Exception, e:
 378                         print "Package matching failed: " + str(e)
 379                         print "Line was: " + line
 380                         sys.exit(1)
 381                 elif line.startswith("application:"):
 382                     thisinfo['name'] = re.match(label_pat, line).group(1)
 383                     if 'icons_src' not in thisinfo:
 384                         thisinfo['icons_src'] = {}
 385                     # Keep path to non-dpi icon in case we need it
 386                     match = re.match(icon_pat_nodpi, line)
 387                     if match:
 388                         thisinfo['icons_src']['-1'] = match.group(1)
 389                 elif line.startswith("launchable-activity:"):
 390                     if 'icons_src' not in thisinfo:
 391                         thisinfo['icons_src'] = {}
 392                     # Only use launchable-activity as fallback to application
 393                     elif '-1' not in thisinfo['icons_src']:
 394                         match = re.match(icon_pat_nodpi, line)
 395                         if match:
 396                             thisinfo['icons_src']['-1'] = match.group(1)
 397                 elif line.startswith("application-icon-"):
 398                     if 'icons_src' not in thisinfo:
 399                         thisinfo['icons_src'] = {}
 400                     match = re.match(icon_pat, line)
 401                     if match:
 402                         density = match.group(1)
 403                         path = match.group(2)
 404                         thisinfo['icons_src'][density] = path
 405                 elif line.startswith("sdkVersion:"):
 406                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 407                 elif line.startswith("native-code:"):
 408                     thisinfo['nativecode'] = []
 409                     for arch in line[13:].split(' '):
 410                         thisinfo['nativecode'].append(arch[1:-1])
 411                 elif line.startswith("uses-permission:"):
 412                     perm = re.match(string_pat, line).group(1)
 413                     if perm.startswith("android.permission."):
 414                         perm = perm[19:]
 415                     thisinfo['permissions'].append(perm)
 416                 elif line.startswith("uses-feature:"):
 417                     perm = re.match(string_pat, line).group(1)
 418                     #Filter out this, it's only added with the latest SDK tools and
 419                     #causes problems for lots of apps.
 420                     if (perm != "android.hardware.screen.portrait" and
 421                         perm != "android.hardware.screen.landscape"):
 422                         if perm.startswith("android.feature."):
 423                             perm = perm[16:]
 424                         thisinfo['features'].append(perm)
 425
 426             if not 'sdkversion' in thisinfo:
 427                 print "  WARNING: no SDK version information found"
 428                 thisinfo['sdkversion'] = 0
 429
 430             # Check for debuggable apks...
 431             if common.isApkDebuggable(apkfile, config):
 432                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 433
 434             # Calculate the sha256...
 435             sha = hashlib.sha256()
 436             with open(apkfile, 'rb') as f:
 437                 while True:
 438                     t = f.read(1024)
 439                     if len(t) == 0:
 440                         break
 441                     sha.update(t)
 442                 thisinfo['sha256'] = sha.hexdigest()
 443
 444             # Get the signature (or md5 of, to be precise)...
 445             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 446             if not os.path.exists(getsig_dir + "/getsig.class"):
 447                 print "ERROR: getsig.class not found. To fix:"
 448                 print "\tcd " + getsig_dir
 449                 print "\t./make.sh"
 450                 sys.exit(1)
 451             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 452                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 453             output = p.communicate()[0]
 454             if options.verbose:
 455                 print output
 456             if p.returncode != 0 or not output.startswith('Result:'):
 457                 print "ERROR: Failed to get apk signature"
 458                 sys.exit(1)
 459             thisinfo['sig'] = output[7:].strip()
 460
 461             iconfilename = "%s.%s.png" % (
 462                     thisinfo['id'],
 463                     thisinfo['versioncode'])
 464
 465             # Extract the icon file...
 466             densities = get_densities()
 467             empty_densities = []
 468             for density in densities:
 469                 if density not in thisinfo['icons_src']:
 470                     empty_densities.append(density)
 471                     continue
 472                 apk = zipfile.ZipFile(apkfile, 'r')
 473                 if 'icons' not in thisinfo:
 474                     thisinfo['icons'] = {}
 475                 iconsrc = thisinfo['icons_src'][density]
 476                 icon_dir = get_icon_dir(repodir, density)
 477                 icondest = os.path.join(icon_dir, iconfilename)
 478
 479                 try:
 480                     iconfile = open(icondest, 'wb')
 481                     iconfile.write(apk.read(iconsrc))
 482                     iconfile.close()
 483                     thisinfo['icons'][density] = iconfilename
 484
 485                 except:
 486                     print "WARNING: Error retrieving icon file"
 487                     del thisinfo['icons'][density]
 488                     del thisinfo['icons_src'][density]
 489                     empty_densities.append(density)
 490
 491                 apk.close()
 492
 493                 resize_icon(icondest, density)
 494
 495             # First try resizing down to not lose quality
 496             last_density = None
 497             for density in densities:
 498                 if density not in empty_densities:
 499                     last_density = density
 500                     continue
 501                 if last_density is None:
 502                     continue
 503                 if options.verbose:
 504                     print "Density %s not available, resizing down from %s" % (
 505                             density, last_density)
 506
 507                 last_iconpath = os.path.join(
 508                         get_icon_dir(repodir, last_density), iconfilename)
 509                 iconpath = os.path.join(
 510                         get_icon_dir(repodir, density), iconfilename)
 511                 im = Image.open(last_iconpath)
 512                 size = launcher_size(density)
 513
 514                 im.thumbnail((size, size), Image.ANTIALIAS)
 515                 im.save(iconpath, "PNG")
 516                 empty_densities.remove(density)
 517
 518             # Then just copy from the highest resolution available
 519             last_density = None
 520             for density in reversed(densities):
 521                 if density not in empty_densities:
 522                     last_density = density
 523                     continue
 524                 if last_density is None:
 525                     continue
 526                 if options.verbose:
 527                     print "Density %s not available, copying from lower density %s" % (
 528                             density, last_density)
 529
 530                 shutil.copyfile(
 531                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 532                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 533
 534                 empty_densities.remove(density)
 535
 536             # Copy from icons-mdpi to icons since mdpi is the baseline density
 537             shutil.copyfile(
 538                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 539                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 540
 541             # Record in known apks, getting the added date at the same time..
 542             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 543             if added:
 544                 thisinfo['added'] = added
 545
 546             apkcache[apkfilename] = thisinfo
 547             cachechanged = True
 548
 549         apks.append(thisinfo)
 550
 551     return apks, cachechanged
 552
 553
 554 repo_pubkey_fingerprint = None
 555
 556 def make_index(apps, apks, repodir, archive, categories):
 557     """Make a repo index.
 558
 559     :param apps: fully populated apps list
 560     :param apks: full populated apks list
 561     :param repodir: the repo directory
 562     :param archive: True if this is the archive repo, False if it's the
 563                     main one.
 564     :param categories: list of categories
 565     """
 566
 567     doc = Document()
 568
 569     def addElement(name, value, doc, parent):
 570         el = doc.createElement(name)
 571         el.appendChild(doc.createTextNode(value))
 572         parent.appendChild(el)
 573     def addElementCDATA(name, value, doc, parent):
 574         el = doc.createElement(name)
 575         el.appendChild(doc.createCDATASection(value))
 576         parent.appendChild(el)
 577
 578     root = doc.createElement("fdroid")
 579     doc.appendChild(root)
 580
 581     repoel = doc.createElement("repo")
 582
 583     if archive:
 584         repoel.setAttribute("name", config['archive_name'])
 585         if config['repo_maxage'] != 0:
 586             repoel.setAttribute("maxage", str(config['repo_maxage']))
 587         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 588         repoel.setAttribute("url", config['archive_url'])
 589         addElement('description', config['archive_description'], doc, repoel)
 590
 591     else:
 592         repoel.setAttribute("name", config['repo_name'])
 593         if config['repo_maxage'] != 0:
 594             repoel.setAttribute("maxage", str(config['repo_maxage']))
 595         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 596         repoel.setAttribute("url", config['repo_url'])
 597         addElement('description', config['repo_description'], doc, repoel)
 598
 599     repoel.setAttribute("version", "10")
 600     repoel.setAttribute("timestamp", str(int(time.time())))
 601
 602     if config['repo_keyalias']:
 603
 604         # Generate a certificate fingerprint the same way keytool does it
 605         # (but with slightly different formatting)
 606         def cert_fingerprint(data):
 607             digest = hashlib.sha1(data).digest()
 608             ret = []
 609             for i in range(4):
 610                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 611             return " ".join(ret)
 612
 613         def extract_pubkey():
 614             p = subprocess.Popen(['keytool', '-exportcert',
 615                                   '-alias', config['repo_keyalias'],
 616                                   '-keystore', config['keystore'],
 617                                   '-storepass', config['keystorepass']],
 618                                  stdout=subprocess.PIPE)
 619             cert = p.communicate()[0]
 620             if p.returncode != 0:
 621                 print "ERROR: Failed to get repo pubkey"
 622                 sys.exit(1)
 623             global repo_pubkey_fingerprint
 624             repo_pubkey_fingerprint = cert_fingerprint(cert)
 625             return "".join("%02x" % ord(b) for b in cert)
 626
 627         repoel.setAttribute("pubkey", extract_pubkey())
 628
 629     root.appendChild(repoel)
 630
 631     for app in apps:
 632
 633         if app['Disabled'] is not None:
 634             continue
 635
 636         # Get a list of the apks for this app...
 637         apklist = []
 638         for apk in apks:
 639             if apk['id'] == app['id']:
 640                 apklist.append(apk)
 641
 642         if len(apklist) == 0:
 643             continue
 644
 645         apel = doc.createElement("application")
 646         apel.setAttribute("id", app['id'])
 647         root.appendChild(apel)
 648
 649         addElement('id', app['id'], doc, apel)
 650         if 'added' in app:
 651             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 652         if 'lastupdated' in app:
 653             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 654         addElement('name', app['Name'], doc, apel)
 655         addElement('summary', app['Summary'], doc, apel)
 656         if app['icon']:
 657             addElement('icon', app['icon'], doc, apel)
 658         def linkres(link):
 659             for app in apps:
 660                 if app['id'] == link:
 661                     return ("fdroid.app:" + link, app['Name'])
 662             raise MetaDataException("Cannot resolve app id " + link)
 663         addElement('desc',
 664                 metadata.description_html(app['Description'], linkres), doc, apel)
 665         addElement('license', app['License'], doc, apel)
 666         if 'Categories' in app:
 667             appcategories = [c.strip() for c in app['Categories'].split(',')]
 668             addElement('categories', ','.join(appcategories), doc, apel)
 669             # We put the first (primary) category in LAST, which will have
 670             # the desired effect of making clients that only understand one
 671             # category see that one.
 672             addElement('category', appcategories[0], doc, apel)
 673         addElement('web', app['Web Site'], doc, apel)
 674         addElement('source', app['Source Code'], doc, apel)
 675         addElement('tracker', app['Issue Tracker'], doc, apel)
 676         if app['Donate']:
 677             addElement('donate', app['Donate'], doc, apel)
 678         if app['Bitcoin']:
 679             addElement('bitcoin', app['Bitcoin'], doc, apel)
 680         if app['Litecoin']:
 681             addElement('litecoin', app['Litecoin'], doc, apel)
 682         if app['Dogecoin']:
 683             addElement('dogecoin', app['Dogecoin'], doc, apel)
 684         if app['FlattrID']:
 685             addElement('flattr', app['FlattrID'], doc, apel)
 686
 687         # These elements actually refer to the current version (i.e. which
 688         # one is recommended. They are historically mis-named, and need
 689         # changing, but stay like this for now to support existing clients.
 690         addElement('marketversion', app['Current Version'], doc, apel)
 691         addElement('marketvercode', app['Current Version Code'], doc, apel)
 692
 693         if app['AntiFeatures']:
 694             af = app['AntiFeatures'].split(',')
 695             # TODO: Temporarily not including UpstreamNonFree in the index,
 696             # because current F-Droid clients do not understand it, and also
 697             # look ugly when they encounter an unknown antifeature. This
 698             # filtering can be removed in time...
 699             if 'UpstreamNonFree' in af:
 700                 af.remove('UpstreamNonFree')
 701             if af:
 702                 addElement('antifeatures', ','.join(af), doc, apel)
 703         if app['Provides']:
 704             pv = app['Provides'].split(',')
 705             addElement('provides', ','.join(pv), doc, apel)
 706         if app['Requires Root']:
 707             addElement('requirements', 'root', doc, apel)
 708
 709         # Sort the apk list into version order, just so the web site
 710         # doesn't have to do any work by default...
 711         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 712
 713         # Check for duplicates - they will make the client unhappy...
 714         for i in range(len(apklist) - 1):
 715             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 716                 print "ERROR - duplicate versions"
 717                 print apklist[i]['apkname']
 718                 print apklist[i+1]['apkname']
 719                 sys.exit(1)
 720
 721         for apk in apklist:
 722             apkel = doc.createElement("package")
 723             apel.appendChild(apkel)
 724             addElement('version', apk['version'], doc, apkel)
 725             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 726             addElement('apkname', apk['apkname'], doc, apkel)
 727             if 'srcname' in apk:
 728                 addElement('srcname', apk['srcname'], doc, apkel)
 729             for hash_type in ['sha256']:
 730                 if not hash_type in apk:
 731                     continue
 732                 hashel = doc.createElement("hash")
 733                 hashel.setAttribute("type", hash_type)
 734                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 735                 apkel.appendChild(hashel)
 736             addElement('sig', apk['sig'], doc, apkel)
 737             addElement('size', str(apk['size']), doc, apkel)
 738             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 739             if 'added' in apk:
 740                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 741             if app['Requires Root']:
 742                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 743                     apk['permissions'].append('ACCESS_SUPERUSER')
 744
 745             if len(apk['permissions']) > 0:
 746                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 747             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 748                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 749             if len(apk['features']) > 0:
 750                 addElement('features', ','.join(apk['features']), doc, apkel)
 751
 752     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 753     if options.pretty:
 754         output = doc.toprettyxml()
 755     else:
 756         output = doc.toxml()
 757     of.write(output)
 758     of.close()
 759
 760     if config['repo_keyalias'] is not None:
 761
 762         if not options.quiet:
 763             print "Creating signed index."
 764             print "Key fingerprint:", repo_pubkey_fingerprint
 765
 766         #Create a jar of the index...
 767         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 768             cwd=repodir, stdout=subprocess.PIPE)
 769         output = p.communicate()[0]
 770         if options.verbose:
 771             print output
 772         if p.returncode != 0:
 773             print "ERROR: Failed to create jar file"
 774             sys.exit(1)
 775
 776         # Sign the index...
 777         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 778             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 779             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 780             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 781         output = p.communicate()[0]
 782         if p.returncode != 0:
 783             print "Failed to sign index"
 784             print output
 785             sys.exit(1)
 786         if options.verbose:
 787             print output
 788
 789     # Copy the repo icon into the repo directory...
 790     icon_dir = os.path.join(repodir ,'icons')
 791     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 792     shutil.copyfile(config['repo_icon'], iconfilename)
 793
 794     # Write a category list in the repo to allow quick access...
 795     catdata = ''
 796     for cat in categories:
 797         catdata += cat + '\n'
 798     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 799     f.write(catdata)
 800     f.close()
 801
 802
 803
 804 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 805
 806     for app in apps:
 807
 808         # Get a list of the apks for this app...
 809         apklist = []
 810         for apk in apks:
 811             if apk['id'] == app['id']:
 812                 apklist.append(apk)
 813
 814         # Sort the apk list into version order...
 815         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 816
 817         if app['Archive Policy']:
 818             keepversions = int(app['Archive Policy'][:-9])
 819         else:
 820             keepversions = defaultkeepversions
 821
 822         if len(apklist) > keepversions:
 823             for apk in apklist[keepversions:]:
 824                 print "Moving " + apk['apkname'] + " to archive"
 825                 shutil.move(os.path.join(repodir, apk['apkname']),
 826                     os.path.join(archivedir, apk['apkname']))
 827                 if 'srcname' in apk:
 828                     shutil.move(os.path.join(repodir, apk['srcname']),
 829                         os.path.join(archivedir, apk['srcname']))
 830                 apks.remove(apk)
 831
 832
 833 config = None
 834 options = None
 835
 836 def main():
 837
 838     global config, options
 839
 840     # Parse command line...
 841     parser = OptionParser()
 842     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 843                       help="Create skeleton metadata files that are missing")
 844     parser.add_option("-v", "--verbose", action="store_true", default=False,
 845                       help="Spew out even more information than normal")
 846     parser.add_option("-q", "--quiet", action="store_true", default=False,
 847                       help="No output, except for warnings and errors")
 848     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 849                       help="Report on build data status")
 850     parser.add_option("-i", "--interactive", default=False, action="store_true",
 851                       help="Interactively ask about things that need updating.")
 852     parser.add_option("-I", "--icons", action="store_true", default=False,
 853                       help="Resize all the icons exceeding the max pixel size and exit")
 854     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 855                       help="Specify editor to use in interactive mode. Default "+
 856                           "is /etc/alternatives/editor")
 857     parser.add_option("-w", "--wiki", default=False, action="store_true",
 858                       help="Update the wiki")
 859     parser.add_option("", "--pretty", action="store_true", default=False,
 860                       help="Produce human-readable index.xml")
 861     parser.add_option("--clean", action="store_true", default=False,
 862                       help="Clean update - don't uses caches, reprocess all apks")
 863     (options, args) = parser.parse_args()
 864
 865     config = common.read_config(options)
 866
 867     repodirs = ['repo']
 868     if config['archive_older'] != 0:
 869         repodirs.append('archive')
 870         if not os.path.exists('archive'):
 871             os.mkdir('archive')
 872
 873     if options.icons:
 874         resize_all_icons(repodirs)
 875         sys.exit(0)
 876
 877     # Get all apps...
 878     apps = metadata.read_metadata()
 879
 880     # Generate a list of categories...
 881     categories = []
 882     for app in apps:
 883         cats = app['Categories'].split(',')
 884         for cat in cats:
 885             if cat not in categories:
 886                 categories.append(cat)
 887
 888     # Read known apks data (will be updated and written back when we've finished)
 889     knownapks = common.KnownApks()
 890
 891     # Gather information about all the apk files in the repo directory, using
 892     # cached data if possible.
 893     apkcachefile = os.path.join('tmp', 'apkcache')
 894     if not options.clean and os.path.exists(apkcachefile):
 895         with open(apkcachefile, 'rb') as cf:
 896             apkcache = pickle.load(cf)
 897     else:
 898         apkcache = {}
 899     cachechanged = False
 900
 901     delete_disabled_builds(apps, apkcache, repodirs)
 902
 903     # Scan all apks in the main repo
 904     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 905     if cc:
 906         cachechanged = True
 907
 908     # Scan the archive repo for apks as well
 909     if len(repodirs) > 1:
 910         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 911         if cc:
 912             cachechanged = True
 913     else:
 914         archapks = []
 915
 916     # Some information from the apks needs to be applied up to the application
 917     # level. When doing this, we use the info from the most recent version's apk.
 918     # We deal with figuring out when the app was added and last updated at the
 919     # same time.
 920     for app in apps:
 921         bestver = 0
 922         added = None
 923         lastupdated = None
 924         for apk in apks + archapks:
 925             if apk['id'] == app['id']:
 926                 if apk['versioncode'] > bestver:
 927                     bestver = apk['versioncode']
 928                     bestapk = apk
 929
 930                 if 'added' in apk:
 931                     if not added or apk['added'] < added:
 932                         added = apk['added']
 933                     if not lastupdated or apk['added'] > lastupdated:
 934                         lastupdated = apk['added']
 935
 936         if added:
 937             app['added'] = added
 938         else:
 939             if options.verbose:
 940                 print "WARNING: Don't know when " + app['id'] + " was added"
 941         if lastupdated:
 942             app['lastupdated'] = lastupdated
 943         else:
 944             if options.verbose:
 945                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 946
 947         if bestver == 0:
 948             if app['Name'] is None:
 949                 app['Name'] = app['id']
 950             app['icon'] = None
 951             if options.verbose and app['Disabled'] is None:
 952                 print "WARNING: Application " + app['id'] + " has no packages"
 953         else:
 954             if app['Name'] is None:
 955                 app['Name'] = bestapk['name']
 956             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 957
 958     # Sort the app list by name, then the web site doesn't have to by default.
 959     # (we had to wait until we'd scanned the apks to do this, because mostly the
 960     # name comes from there!)
 961     apps = sorted(apps, key=lambda app: app['Name'].upper())
 962
 963     # Generate warnings for apk's with no metadata (or create skeleton
 964     # metadata files, if requested on the command line)
 965     for apk in apks:
 966         found = False
 967         for app in apps:
 968             if app['id'] == apk['id']:
 969                 found = True
 970                 break
 971         if not found:
 972             if options.createmeta:
 973                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 974                 f.write("License:Unknown\n")
 975                 f.write("Web Site:\n")
 976                 f.write("Source Code:\n")
 977                 f.write("Issue Tracker:\n")
 978                 f.write("Summary:" + apk['name'] + "\n")
 979                 f.write("Description:\n")
 980                 f.write(apk['name'] + "\n")
 981                 f.write(".\n")
 982                 f.close()
 983                 print "Generated skeleton metadata for " + apk['id']
 984             else:
 985                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 986                 print "       " + apk['name'] + " - " + apk['version']
 987
 988     if len(repodirs) > 1:
 989         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 990
 991     # Make the index for the main repo...
 992     make_index(apps, apks, repodirs[0], False, categories)
 993
 994     # If there's an archive repo,  make the index for it. We already scanned it
 995     # earlier on.
 996     if len(repodirs) > 1:
 997         make_index(apps, archapks, repodirs[1], True, categories)
 998
 999     if config['update_stats']:
1000
1001         # Update known apks info...
1002         knownapks.writeifchanged()
1003
1004         # Generate latest apps data for widget
1005         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1006             data = ''
1007             for line in file(os.path.join('stats', 'latestapps.txt')):
1008                 appid = line.rstrip()
1009                 data += appid + "\t"
1010                 for app in apps:
1011                     if app['id'] == appid:
1012                         data += app['Name'] + "\t"
1013                         if app['icon'] is not None:
1014                             data += app['icon'] + "\t"
1015                         data += app['License'] + "\n"
1016                         break
1017             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1018             f.write(data)
1019             f.close()
1020
1021     if cachechanged:
1022         with open(apkcachefile, 'wb') as cf:
1023             pickle.dump(apkcache, cf)
1024
1025     # Update the wiki...
1026     if options.wiki:
1027         update_wiki(apps, apks + archapks)
1028
1029     print "Finished."
1030
1031 if __name__ == "__main__":
1032     main()
1033