fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def dpi_to_px(density):
  42     return (int(density) * 48) / 160
  43
  44 def px_to_dpi(px):
  45     return (int(px) * 160) / 48
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         elif validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         elif cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         elif buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 191             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 192
 193         # We can't have underscores in the page name, even if they're in
 194         # the package ID, because MediaWiki messes with them...
 195         pagename = app['id'].replace('_', ' ')
 196
 197         # Drop a trailing newline, because mediawiki is going to drop it anyway
 198         # and it we don't we'll think the page has changed when it hasn't...
 199         if wikidata.endswith('\n'):
 200             wikidata = wikidata[:-1]
 201
 202         generated_pages[pagename] = wikidata
 203
 204         # Make a redirect from the name to the ID too, unless there's
 205         # already an existing page with the name and it isn't a redirect.
 206         noclobber = False
 207         apppagename = app['Name'].replace('_', ' ')
 208         apppagename = apppagename.replace('{', '')
 209         apppagename = apppagename.replace('}', ' ')
 210         apppagename = apppagename.replace(':', ' ')
 211         # Drop double spaces caused mostly by replacing ':' above
 212         apppagename = apppagename.replace('  ', ' ')
 213         for expagename in site.allpages(prefix=apppagename,
 214                 filterredir='nonredirects', generator=False):
 215             if expagename == apppagename:
 216                 noclobber = True
 217         # Another reason not to make the redirect page is if the app name
 218         # is the same as it's ID, because that will overwrite the real page
 219         # with an redirect to itself! (Although it seems like an odd
 220         # scenario this happens a lot, e.g. where there is metadata but no
 221         # builds or binaries to extract a name from.
 222         if apppagename == pagename:
 223             noclobber = True
 224         if not noclobber:
 225             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 226
 227     for tcat, genp in [(wikicat, generated_pages),
 228             (wikiredircat, generated_redirects)]:
 229         catpages = site.Pages['Category:' + tcat]
 230         existingpages = []
 231         for page in catpages:
 232             existingpages.append(page.name)
 233             if page.name in genp:
 234                 pagetxt = page.edit()
 235                 if pagetxt != genp[page.name]:
 236                     print "Updating modified page " + page.name
 237                     page.save(genp[page.name], summary='Auto-updated')
 238                 else:
 239                     if options.verbose:
 240                         print "Page " + page.name + " is unchanged"
 241             else:
 242                 print "Deleting page " + page.name
 243                 page.delete('No longer published')
 244         for pagename, text in genp.items():
 245             if options.verbose:
 246                 print "Checking " + pagename
 247             if not pagename in existingpages:
 248                 print "Creating page " + pagename
 249                 try:
 250                     newpage = site.Pages[pagename]
 251                     newpage.save(text, summary='Auto-created')
 252                 except:
 253                     print "...FAILED to create page"
 254
 255     # Purge server cache to ensure counts are up to date
 256     site.pages['Repository Maintenance'].purge()
 257
 258 def delete_disabled_builds(apps, apkcache, repodirs):
 259     """Delete disabled build outputs.
 260
 261     :param apps: list of all applications, as per metadata.read_metadata
 262     :param apkcache: current apk cache information
 263     :param repodirs: the repo directories to process
 264     """
 265     for app in apps:
 266         for build in app['builds']:
 267             if 'disable' in build:
 268                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 269                 for repodir in repodirs:
 270                     apkpath = os.path.join(repodir, apkfilename)
 271                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 272                     for name in [apkpath, srcpath]:
 273                         if os.path.exists(name):
 274                             print "Deleting disabled build output " + apkfilename
 275                             os.remove(name)
 276                 if apkfilename in apkcache:
 277                     del apkcache[apkfilename]
 278
 279 def resize_icon(iconpath, density):
 280
 281     try:
 282         im = Image.open(iconpath)
 283         size = dpi_to_px(density)
 284
 285         if any(length > size for length in im.size):
 286             oldsize = im.size
 287             im.thumbnail((size, size), Image.ANTIALIAS)
 288             print iconpath, "was too large at", oldsize, "- new size is", im.size
 289             im.save(iconpath, "PNG")
 290
 291         else:
 292             if options.verbose:
 293                 print iconpath, "is small enough:", im.size
 294
 295     except Exception,e:
 296         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 297
 298 def resize_all_icons(repodirs):
 299     """Resize all icons that exceed the max size
 300
 301     :param repodirs: the repo directories to process
 302     """
 303     for repodir in repodirs:
 304         for density in get_densities():
 305             icon_dir = get_icon_dir(repodir, density)
 306             icon_glob = os.path.join(icon_dir, '*.png')
 307             for iconpath in glob.glob(icon_glob):
 308                 resize_icon(iconpath, density)
 309
 310 def scan_apks(apps, apkcache, repodir, knownapks):
 311     """Scan the apks in the given repo directory.
 312
 313     This also extracts the icons.
 314
 315     :param apps: list of all applications, as per metadata.read_metadata
 316     :param apkcache: current apk cache information
 317     :param repodir: repo directory to scan
 318     :param knownapks: known apks info
 319     :returns: (apks, cachechanged) where apks is a list of apk information,
 320               and cachechanged is True if the apkcache got changed.
 321     """
 322
 323     cachechanged = False
 324
 325     icon_dirs = get_icon_dirs(repodir)
 326     for icon_dir in icon_dirs:
 327         if os.path.exists(icon_dir):
 328             if options.clean:
 329                 shutil.rmtree(icon_dir)
 330                 os.makedirs(icon_dir)
 331         else:
 332             os.makedirs(icon_dir)
 333
 334     apks = []
 335     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 336     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 337     vername_pat = re.compile(".*versionName='([^']*)'.*")
 338     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 339     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 340     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 341     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 342     string_pat = re.compile(".*'([^']*)'.*")
 343     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 344
 345         apkfilename = apkfile[len(repodir) + 1:]
 346         if ' ' in apkfilename:
 347             print "No spaces in APK filenames!"
 348             sys.exit(1)
 349
 350         if apkfilename in apkcache:
 351             if options.verbose:
 352                 print "Reading " + apkfilename + " from cache"
 353             thisinfo = apkcache[apkfilename]
 354
 355         else:
 356
 357             if not options.quiet:
 358                 print "Processing " + apkfilename
 359             thisinfo = {}
 360             thisinfo['apkname'] = apkfilename
 361             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 362             if os.path.exists(os.path.join(repodir, srcfilename)):
 363                 thisinfo['srcname'] = srcfilename
 364             thisinfo['size'] = os.path.getsize(apkfile)
 365             thisinfo['permissions'] = []
 366             thisinfo['features'] = []
 367             thisinfo['icons_src'] = {}
 368             thisinfo['icons'] = {}
 369             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 370                                   'dump', 'badging', apkfile],
 371                                  stdout=subprocess.PIPE)
 372             output = p.communicate()[0]
 373             if p.returncode != 0:
 374                 print "ERROR: Failed to get apk information"
 375                 sys.exit(1)
 376             for line in output.splitlines():
 377                 if line.startswith("package:"):
 378                     try:
 379                         thisinfo['id'] = re.match(name_pat, line).group(1)
 380                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 381                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 382                     except Exception, e:
 383                         print "Package matching failed: " + str(e)
 384                         print "Line was: " + line
 385                         sys.exit(1)
 386                 elif line.startswith("application:"):
 387                     thisinfo['name'] = re.match(label_pat, line).group(1)
 388                     # Keep path to non-dpi icon in case we need it
 389                     match = re.match(icon_pat_nodpi, line)
 390                     if match:
 391                         thisinfo['icons_src']['-1'] = match.group(1)
 392                 elif line.startswith("launchable-activity:"):
 393                     # Only use launchable-activity as fallback to application
 394                     if '-1' not in thisinfo['icons_src']:
 395                         match = re.match(icon_pat_nodpi, line)
 396                         if match:
 397                             thisinfo['icons_src']['-1'] = match.group(1)
 398                 elif line.startswith("application-icon-"):
 399                     match = re.match(icon_pat, line)
 400                     if match:
 401                         density = match.group(1)
 402                         path = match.group(2)
 403                         thisinfo['icons_src'][density] = path
 404                 elif line.startswith("sdkVersion:"):
 405                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 406                 elif line.startswith("native-code:"):
 407                     thisinfo['nativecode'] = []
 408                     for arch in line[13:].split(' '):
 409                         thisinfo['nativecode'].append(arch[1:-1])
 410                 elif line.startswith("uses-permission:"):
 411                     perm = re.match(string_pat, line).group(1)
 412                     if perm.startswith("android.permission."):
 413                         perm = perm[19:]
 414                     thisinfo['permissions'].append(perm)
 415                 elif line.startswith("uses-feature:"):
 416                     perm = re.match(string_pat, line).group(1)
 417                     #Filter out this, it's only added with the latest SDK tools and
 418                     #causes problems for lots of apps.
 419                     if (perm != "android.hardware.screen.portrait" and
 420                         perm != "android.hardware.screen.landscape"):
 421                         if perm.startswith("android.feature."):
 422                             perm = perm[16:]
 423                         thisinfo['features'].append(perm)
 424
 425             if not 'sdkversion' in thisinfo:
 426                 print "  WARNING: no SDK version information found"
 427                 thisinfo['sdkversion'] = 0
 428
 429             # Check for debuggable apks...
 430             if common.isApkDebuggable(apkfile, config):
 431                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 432
 433             # Calculate the sha256...
 434             sha = hashlib.sha256()
 435             with open(apkfile, 'rb') as f:
 436                 while True:
 437                     t = f.read(1024)
 438                     if len(t) == 0:
 439                         break
 440                     sha.update(t)
 441                 thisinfo['sha256'] = sha.hexdigest()
 442
 443             # Get the signature (or md5 of, to be precise)...
 444             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 445             if not os.path.exists(getsig_dir + "/getsig.class"):
 446                 print "ERROR: getsig.class not found. To fix:"
 447                 print "\tcd " + getsig_dir
 448                 print "\t./make.sh"
 449                 sys.exit(1)
 450             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 451                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 452             output = p.communicate()[0]
 453             if options.verbose:
 454                 print output
 455             if p.returncode != 0 or not output.startswith('Result:'):
 456                 print "ERROR: Failed to get apk signature"
 457                 sys.exit(1)
 458             thisinfo['sig'] = output[7:].strip()
 459
 460             apk = zipfile.ZipFile(apkfile, 'r')
 461
 462             iconfilename = "%s.%s.png" % (
 463                     thisinfo['id'],
 464                     thisinfo['versioncode'])
 465
 466             # Extract the icon file...
 467             densities = get_densities()
 468             empty_densities = []
 469             for density in densities:
 470                 if density not in thisinfo['icons_src']:
 471                     empty_densities.append(density)
 472                     continue
 473                 iconsrc = thisinfo['icons_src'][density]
 474                 icon_dir = get_icon_dir(repodir, density)
 475                 icondest = os.path.join(icon_dir, iconfilename)
 476
 477                 try:
 478                     iconfile = open(icondest, 'wb')
 479                     iconfile.write(apk.read(iconsrc))
 480                     iconfile.close()
 481                     thisinfo['icons'][density] = iconfilename
 482
 483                 except:
 484                     print "WARNING: Error retrieving icon file"
 485                     del thisinfo['icons'][density]
 486                     del thisinfo['icons_src'][density]
 487                     empty_densities.append(density)
 488
 489             if '-1' in thisinfo['icons_src']:
 490                 iconsrc = thisinfo['icons_src']['-1']
 491                 iconpath = os.path.join(
 492                         get_icon_dir(repodir, None), iconfilename)
 493                 iconfile = open(iconpath, 'wb')
 494                 iconfile.write(apk.read(iconsrc))
 495                 iconfile.close()
 496                 im = Image.open(iconpath)
 497                 dpi = px_to_dpi(im.size[0])
 498                 for density in densities:
 499                     if density in thisinfo['icons']:
 500                         break
 501                     if dpi >= int(density):
 502                         thisinfo['icons'][density] = iconfilename
 503                         shutil.move(iconpath,
 504                                 os.path.join(get_icon_dir(repodir, density), iconfilename))
 505                         empty_densities.remove(density)
 506                         break
 507
 508             apk.close()
 509
 510             # First try resizing down to not lose quality
 511             last_density = None
 512             for density in densities:
 513                 if density not in empty_densities:
 514                     last_density = density
 515                     continue
 516                 if last_density is None:
 517                     continue
 518                 if options.verbose:
 519                     print "Density %s not available, resizing down from %s" % (
 520                             density, last_density)
 521
 522                 last_iconpath = os.path.join(
 523                         get_icon_dir(repodir, last_density), iconfilename)
 524                 iconpath = os.path.join(
 525                         get_icon_dir(repodir, density), iconfilename)
 526                 im = Image.open(last_iconpath)
 527                 size = dpi_to_px(density)
 528
 529                 im.thumbnail((size, size), Image.ANTIALIAS)
 530                 im.save(iconpath, "PNG")
 531                 empty_densities.remove(density)
 532
 533             # Then just copy from the highest resolution available
 534             last_density = None
 535             for density in reversed(densities):
 536                 if density not in empty_densities:
 537                     last_density = density
 538                     continue
 539                 if last_density is None:
 540                     continue
 541                 if options.verbose:
 542                     print "Density %s not available, copying from lower density %s" % (
 543                             density, last_density)
 544
 545                 shutil.copyfile(
 546                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 547                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 548
 549                 empty_densities.remove(density)
 550
 551             for density in densities:
 552                 icon_dir = get_icon_dir(repodir, density)
 553                 icondest = os.path.join(icon_dir, iconfilename)
 554                 resize_icon(icondest, density)
 555
 556             # Copy from icons-mdpi to icons since mdpi is the baseline density
 557             shutil.copyfile(
 558                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 559                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 560
 561             # Record in known apks, getting the added date at the same time..
 562             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 563             if added:
 564                 thisinfo['added'] = added
 565
 566             apkcache[apkfilename] = thisinfo
 567             cachechanged = True
 568
 569         apks.append(thisinfo)
 570
 571     return apks, cachechanged
 572
 573
 574 repo_pubkey_fingerprint = None
 575
 576 def make_index(apps, apks, repodir, archive, categories):
 577     """Make a repo index.
 578
 579     :param apps: fully populated apps list
 580     :param apks: full populated apks list
 581     :param repodir: the repo directory
 582     :param archive: True if this is the archive repo, False if it's the
 583                     main one.
 584     :param categories: list of categories
 585     """
 586
 587     doc = Document()
 588
 589     def addElement(name, value, doc, parent):
 590         el = doc.createElement(name)
 591         el.appendChild(doc.createTextNode(value))
 592         parent.appendChild(el)
 593     def addElementCDATA(name, value, doc, parent):
 594         el = doc.createElement(name)
 595         el.appendChild(doc.createCDATASection(value))
 596         parent.appendChild(el)
 597
 598     root = doc.createElement("fdroid")
 599     doc.appendChild(root)
 600
 601     repoel = doc.createElement("repo")
 602
 603     if archive:
 604         repoel.setAttribute("name", config['archive_name'])
 605         if config['repo_maxage'] != 0:
 606             repoel.setAttribute("maxage", str(config['repo_maxage']))
 607         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 608         repoel.setAttribute("url", config['archive_url'])
 609         addElement('description', config['archive_description'], doc, repoel)
 610
 611     else:
 612         repoel.setAttribute("name", config['repo_name'])
 613         if config['repo_maxage'] != 0:
 614             repoel.setAttribute("maxage", str(config['repo_maxage']))
 615         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 616         repoel.setAttribute("url", config['repo_url'])
 617         addElement('description', config['repo_description'], doc, repoel)
 618
 619     repoel.setAttribute("version", "10")
 620     repoel.setAttribute("timestamp", str(int(time.time())))
 621
 622     if config['repo_keyalias']:
 623
 624         # Generate a certificate fingerprint the same way keytool does it
 625         # (but with slightly different formatting)
 626         def cert_fingerprint(data):
 627             digest = hashlib.sha1(data).digest()
 628             ret = []
 629             for i in range(4):
 630                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 631             return " ".join(ret)
 632
 633         def extract_pubkey():
 634             p = subprocess.Popen(['keytool', '-exportcert',
 635                                   '-alias', config['repo_keyalias'],
 636                                   '-keystore', config['keystore'],
 637                                   '-storepass', config['keystorepass']],
 638                                  stdout=subprocess.PIPE)
 639             cert = p.communicate()[0]
 640             if p.returncode != 0:
 641                 print "ERROR: Failed to get repo pubkey"
 642                 sys.exit(1)
 643             global repo_pubkey_fingerprint
 644             repo_pubkey_fingerprint = cert_fingerprint(cert)
 645             return "".join("%02x" % ord(b) for b in cert)
 646
 647         repoel.setAttribute("pubkey", extract_pubkey())
 648
 649     root.appendChild(repoel)
 650
 651     for app in apps:
 652
 653         if app['Disabled'] is not None:
 654             continue
 655
 656         # Get a list of the apks for this app...
 657         apklist = []
 658         for apk in apks:
 659             if apk['id'] == app['id']:
 660                 apklist.append(apk)
 661
 662         if len(apklist) == 0:
 663             continue
 664
 665         apel = doc.createElement("application")
 666         apel.setAttribute("id", app['id'])
 667         root.appendChild(apel)
 668
 669         addElement('id', app['id'], doc, apel)
 670         if 'added' in app:
 671             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 672         if 'lastupdated' in app:
 673             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 674         addElement('name', app['Name'], doc, apel)
 675         addElement('summary', app['Summary'], doc, apel)
 676         if app['icon']:
 677             addElement('icon', app['icon'], doc, apel)
 678         def linkres(link):
 679             for app in apps:
 680                 if app['id'] == link:
 681                     return ("fdroid.app:" + link, app['Name'])
 682             raise MetaDataException("Cannot resolve app id " + link)
 683         addElement('desc',
 684                 metadata.description_html(app['Description'], linkres), doc, apel)
 685         addElement('license', app['License'], doc, apel)
 686         if 'Categories' in app:
 687             appcategories = [c.strip() for c in app['Categories'].split(',')]
 688             addElement('categories', ','.join(appcategories), doc, apel)
 689             # We put the first (primary) category in LAST, which will have
 690             # the desired effect of making clients that only understand one
 691             # category see that one.
 692             addElement('category', appcategories[0], doc, apel)
 693         addElement('web', app['Web Site'], doc, apel)
 694         addElement('source', app['Source Code'], doc, apel)
 695         addElement('tracker', app['Issue Tracker'], doc, apel)
 696         if app['Donate']:
 697             addElement('donate', app['Donate'], doc, apel)
 698         if app['Bitcoin']:
 699             addElement('bitcoin', app['Bitcoin'], doc, apel)
 700         if app['Litecoin']:
 701             addElement('litecoin', app['Litecoin'], doc, apel)
 702         if app['Dogecoin']:
 703             addElement('dogecoin', app['Dogecoin'], doc, apel)
 704         if app['FlattrID']:
 705             addElement('flattr', app['FlattrID'], doc, apel)
 706
 707         # These elements actually refer to the current version (i.e. which
 708         # one is recommended. They are historically mis-named, and need
 709         # changing, but stay like this for now to support existing clients.
 710         addElement('marketversion', app['Current Version'], doc, apel)
 711         addElement('marketvercode', app['Current Version Code'], doc, apel)
 712
 713         if app['AntiFeatures']:
 714             af = app['AntiFeatures'].split(',')
 715             # TODO: Temporarily not including UpstreamNonFree in the index,
 716             # because current F-Droid clients do not understand it, and also
 717             # look ugly when they encounter an unknown antifeature. This
 718             # filtering can be removed in time...
 719             if 'UpstreamNonFree' in af:
 720                 af.remove('UpstreamNonFree')
 721             if af:
 722                 addElement('antifeatures', ','.join(af), doc, apel)
 723         if app['Provides']:
 724             pv = app['Provides'].split(',')
 725             addElement('provides', ','.join(pv), doc, apel)
 726         if app['Requires Root']:
 727             addElement('requirements', 'root', doc, apel)
 728
 729         # Sort the apk list into version order, just so the web site
 730         # doesn't have to do any work by default...
 731         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 732
 733         # Check for duplicates - they will make the client unhappy...
 734         for i in range(len(apklist) - 1):
 735             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 736                 print "ERROR - duplicate versions"
 737                 print apklist[i]['apkname']
 738                 print apklist[i+1]['apkname']
 739                 sys.exit(1)
 740
 741         for apk in apklist:
 742             apkel = doc.createElement("package")
 743             apel.appendChild(apkel)
 744             addElement('version', apk['version'], doc, apkel)
 745             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 746             addElement('apkname', apk['apkname'], doc, apkel)
 747             if 'srcname' in apk:
 748                 addElement('srcname', apk['srcname'], doc, apkel)
 749             for hash_type in ['sha256']:
 750                 if not hash_type in apk:
 751                     continue
 752                 hashel = doc.createElement("hash")
 753                 hashel.setAttribute("type", hash_type)
 754                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 755                 apkel.appendChild(hashel)
 756             addElement('sig', apk['sig'], doc, apkel)
 757             addElement('size', str(apk['size']), doc, apkel)
 758             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 759             if 'added' in apk:
 760                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 761             if app['Requires Root']:
 762                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 763                     apk['permissions'].append('ACCESS_SUPERUSER')
 764
 765             if len(apk['permissions']) > 0:
 766                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 767             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 768                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 769             if len(apk['features']) > 0:
 770                 addElement('features', ','.join(apk['features']), doc, apkel)
 771
 772     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 773     if options.pretty:
 774         output = doc.toprettyxml()
 775     else:
 776         output = doc.toxml()
 777     of.write(output)
 778     of.close()
 779
 780     if config['repo_keyalias'] is not None:
 781
 782         if not options.quiet:
 783             print "Creating signed index."
 784             print "Key fingerprint:", repo_pubkey_fingerprint
 785
 786         #Create a jar of the index...
 787         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 788             cwd=repodir, stdout=subprocess.PIPE)
 789         output = p.communicate()[0]
 790         if options.verbose:
 791             print output
 792         if p.returncode != 0:
 793             print "ERROR: Failed to create jar file"
 794             sys.exit(1)
 795
 796         # Sign the index...
 797         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 798             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 799             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 800             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 801         output = p.communicate()[0]
 802         if p.returncode != 0:
 803             print "Failed to sign index"
 804             print output
 805             sys.exit(1)
 806         if options.verbose:
 807             print output
 808
 809     # Copy the repo icon into the repo directory...
 810     icon_dir = os.path.join(repodir ,'icons')
 811     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 812     shutil.copyfile(config['repo_icon'], iconfilename)
 813
 814     # Write a category list in the repo to allow quick access...
 815     catdata = ''
 816     for cat in categories:
 817         catdata += cat + '\n'
 818     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 819     f.write(catdata)
 820     f.close()
 821
 822
 823
 824 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 825
 826     for app in apps:
 827
 828         # Get a list of the apks for this app...
 829         apklist = []
 830         for apk in apks:
 831             if apk['id'] == app['id']:
 832                 apklist.append(apk)
 833
 834         # Sort the apk list into version order...
 835         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 836
 837         if app['Archive Policy']:
 838             keepversions = int(app['Archive Policy'][:-9])
 839         else:
 840             keepversions = defaultkeepversions
 841
 842         if len(apklist) > keepversions:
 843             for apk in apklist[keepversions:]:
 844                 print "Moving " + apk['apkname'] + " to archive"
 845                 shutil.move(os.path.join(repodir, apk['apkname']),
 846                     os.path.join(archivedir, apk['apkname']))
 847                 if 'srcname' in apk:
 848                     shutil.move(os.path.join(repodir, apk['srcname']),
 849                         os.path.join(archivedir, apk['srcname']))
 850                 apks.remove(apk)
 851
 852
 853 config = None
 854 options = None
 855
 856 def main():
 857
 858     global config, options
 859
 860     # Parse command line...
 861     parser = OptionParser()
 862     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 863                       help="Create skeleton metadata files that are missing")
 864     parser.add_option("-v", "--verbose", action="store_true", default=False,
 865                       help="Spew out even more information than normal")
 866     parser.add_option("-q", "--quiet", action="store_true", default=False,
 867                       help="No output, except for warnings and errors")
 868     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 869                       help="Report on build data status")
 870     parser.add_option("-i", "--interactive", default=False, action="store_true",
 871                       help="Interactively ask about things that need updating.")
 872     parser.add_option("-I", "--icons", action="store_true", default=False,
 873                       help="Resize all the icons exceeding the max pixel size and exit")
 874     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 875                       help="Specify editor to use in interactive mode. Default "+
 876                           "is /etc/alternatives/editor")
 877     parser.add_option("-w", "--wiki", default=False, action="store_true",
 878                       help="Update the wiki")
 879     parser.add_option("", "--pretty", action="store_true", default=False,
 880                       help="Produce human-readable index.xml")
 881     parser.add_option("--clean", action="store_true", default=False,
 882                       help="Clean update - don't uses caches, reprocess all apks")
 883     (options, args) = parser.parse_args()
 884
 885     config = common.read_config(options)
 886
 887     repodirs = ['repo']
 888     if config['archive_older'] != 0:
 889         repodirs.append('archive')
 890         if not os.path.exists('archive'):
 891             os.mkdir('archive')
 892
 893     if options.icons:
 894         resize_all_icons(repodirs)
 895         sys.exit(0)
 896
 897     # Get all apps...
 898     apps = metadata.read_metadata()
 899
 900     # Generate a list of categories...
 901     categories = []
 902     for app in apps:
 903         cats = app['Categories'].split(',')
 904         for cat in cats:
 905             if cat not in categories:
 906                 categories.append(cat)
 907
 908     # Read known apks data (will be updated and written back when we've finished)
 909     knownapks = common.KnownApks()
 910
 911     # Gather information about all the apk files in the repo directory, using
 912     # cached data if possible.
 913     apkcachefile = os.path.join('tmp', 'apkcache')
 914     if not options.clean and os.path.exists(apkcachefile):
 915         with open(apkcachefile, 'rb') as cf:
 916             apkcache = pickle.load(cf)
 917     else:
 918         apkcache = {}
 919     cachechanged = False
 920
 921     delete_disabled_builds(apps, apkcache, repodirs)
 922
 923     # Scan all apks in the main repo
 924     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 925     if cc:
 926         cachechanged = True
 927
 928     # Scan the archive repo for apks as well
 929     if len(repodirs) > 1:
 930         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 931         if cc:
 932             cachechanged = True
 933     else:
 934         archapks = []
 935
 936     # Some information from the apks needs to be applied up to the application
 937     # level. When doing this, we use the info from the most recent version's apk.
 938     # We deal with figuring out when the app was added and last updated at the
 939     # same time.
 940     for app in apps:
 941         bestver = 0
 942         added = None
 943         lastupdated = None
 944         for apk in apks + archapks:
 945             if apk['id'] == app['id']:
 946                 if apk['versioncode'] > bestver:
 947                     bestver = apk['versioncode']
 948                     bestapk = apk
 949
 950                 if 'added' in apk:
 951                     if not added or apk['added'] < added:
 952                         added = apk['added']
 953                     if not lastupdated or apk['added'] > lastupdated:
 954                         lastupdated = apk['added']
 955
 956         if added:
 957             app['added'] = added
 958         else:
 959             if options.verbose:
 960                 print "WARNING: Don't know when " + app['id'] + " was added"
 961         if lastupdated:
 962             app['lastupdated'] = lastupdated
 963         else:
 964             if options.verbose:
 965                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 966
 967         if bestver == 0:
 968             if app['Name'] is None:
 969                 app['Name'] = app['id']
 970             app['icon'] = None
 971             if options.verbose and app['Disabled'] is None:
 972                 print "WARNING: Application " + app['id'] + " has no packages"
 973         else:
 974             if app['Name'] is None:
 975                 app['Name'] = bestapk['name']
 976             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 977
 978     # Sort the app list by name, then the web site doesn't have to by default.
 979     # (we had to wait until we'd scanned the apks to do this, because mostly the
 980     # name comes from there!)
 981     apps = sorted(apps, key=lambda app: app['Name'].upper())
 982
 983     # Generate warnings for apk's with no metadata (or create skeleton
 984     # metadata files, if requested on the command line)
 985     for apk in apks:
 986         found = False
 987         for app in apps:
 988             if app['id'] == apk['id']:
 989                 found = True
 990                 break
 991         if not found:
 992             if options.createmeta:
 993                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 994                 f.write("License:Unknown\n")
 995                 f.write("Web Site:\n")
 996                 f.write("Source Code:\n")
 997                 f.write("Issue Tracker:\n")
 998                 f.write("Summary:" + apk['name'] + "\n")
 999                 f.write("Description:\n")
1000                 f.write(apk['name'] + "\n")
1001                 f.write(".\n")
1002                 f.close()
1003                 print "Generated skeleton metadata for " + apk['id']
1004             else:
1005                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
1006                 print "       " + apk['name'] + " - " + apk['version']
1007
1008     if len(repodirs) > 1:
1009         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
1010
1011     # Make the index for the main repo...
1012     make_index(apps, apks, repodirs[0], False, categories)
1013
1014     # If there's an archive repo,  make the index for it. We already scanned it
1015     # earlier on.
1016     if len(repodirs) > 1:
1017         make_index(apps, archapks, repodirs[1], True, categories)
1018
1019     if config['update_stats']:
1020
1021         # Update known apks info...
1022         knownapks.writeifchanged()
1023
1024         # Generate latest apps data for widget
1025         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1026             data = ''
1027             for line in file(os.path.join('stats', 'latestapps.txt')):
1028                 appid = line.rstrip()
1029                 data += appid + "\t"
1030                 for app in apps:
1031                     if app['id'] == appid:
1032                         data += app['Name'] + "\t"
1033                         if app['icon'] is not None:
1034                             data += app['icon'] + "\t"
1035                         data += app['License'] + "\n"
1036                         break
1037             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1038             f.write(data)
1039             f.close()
1040
1041     if cachechanged:
1042         with open(apkcachefile, 'wb') as cf:
1043             pickle.dump(apkcache, cf)
1044
1045     # Update the wiki...
1046     if options.wiki:
1047         update_wiki(apps, apks + archapks)
1048
1049     print "Finished."
1050
1051 if __name__ == "__main__":
1052     main()
1053