fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         if app['Current Version Code'] == '0':
 145             cantupdate = True
 146         # Sort with most recent first...
 147         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 148
 149         wikidata += "=Versions=\n"
 150         if len(apklist) == 0:
 151             wikidata += "We currently have no versions of this app available."
 152         elif not gotcurrentver:
 153             wikidata += "We don't have the current version of this app."
 154         else:
 155             wikidata += "We have the current version of this app."
 156         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 157         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 158         if len(app['No Source Since']) > 0:
 159             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 160         if len(app['Current Version']) > 0:
 161             wikidata += "The current (recommended) version is " + app['Current Version']
 162             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 163         validapks = 0
 164         for apk in apklist:
 165             wikidata += "==" + apk['version'] + "==\n"
 166
 167             if 'buildproblem' in apk:
 168                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 169             else:
 170                 validapks += 1
 171                 wikidata += "This version is built and signed by "
 172                 if 'srcname' in apk:
 173                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 174                 else:
 175                     wikidata += "the original developer.\n\n"
 176             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 177
 178         wikidata += '\n[[Category:' + wikicat + ']]\n'
 179         if len(app['No Source Since']) > 0:
 180             wikidata += '\n[[Category:Apps missing source code]]\n'
 181         if validapks == 0 and not app['Disabled']:
 182             wikidata += '\n[[Category:Apps with no packages]]\n'
 183         if cantupdate and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps we can't update]]\n"
 185         if buildfails and not app['Disabled']:
 186             wikidata += "\n[[Category:Apps with failing builds]]\n"
 187         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 188             wikidata += '\n[[Category:Apps to Update]]\n'
 189         if app['Disabled']:
 190             wikidata += '\n[[Category:Apps that are disabled]]\n'
 191         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 192             wikidata += '\n[[Category:Apps with no update check]]\n'
 193         for appcat in app['Categories']:
 194             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 195
 196         # We can't have underscores in the page name, even if they're in
 197         # the package ID, because MediaWiki messes with them...
 198         pagename = app['id'].replace('_', ' ')
 199
 200         # Drop a trailing newline, because mediawiki is going to drop it anyway
 201         # and it we don't we'll think the page has changed when it hasn't...
 202         if wikidata.endswith('\n'):
 203             wikidata = wikidata[:-1]
 204
 205         generated_pages[pagename] = wikidata
 206
 207         # Make a redirect from the name to the ID too, unless there's
 208         # already an existing page with the name and it isn't a redirect.
 209         noclobber = False
 210         apppagename = app['Name'].replace('_', ' ')
 211         apppagename = apppagename.replace('{', '')
 212         apppagename = apppagename.replace('}', ' ')
 213         apppagename = apppagename.replace(':', ' ')
 214         # Drop double spaces caused mostly by replacing ':' above
 215         apppagename = apppagename.replace('  ', ' ')
 216         for expagename in site.allpages(prefix=apppagename,
 217                 filterredir='nonredirects', generator=False):
 218             if expagename == apppagename:
 219                 noclobber = True
 220         # Another reason not to make the redirect page is if the app name
 221         # is the same as it's ID, because that will overwrite the real page
 222         # with an redirect to itself! (Although it seems like an odd
 223         # scenario this happens a lot, e.g. where there is metadata but no
 224         # builds or binaries to extract a name from.
 225         if apppagename == pagename:
 226             noclobber = True
 227         if not noclobber:
 228             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 229
 230     for tcat, genp in [(wikicat, generated_pages),
 231             (wikiredircat, generated_redirects)]:
 232         catpages = site.Pages['Category:' + tcat]
 233         existingpages = []
 234         for page in catpages:
 235             existingpages.append(page.name)
 236             if page.name in genp:
 237                 pagetxt = page.edit()
 238                 if pagetxt != genp[page.name]:
 239                     logging.info("Updating modified page " + page.name)
 240                     page.save(genp[page.name], summary='Auto-updated')
 241                 else:
 242                     logging.info("Page " + page.name + " is unchanged")
 243             else:
 244                 logging.info("Deleting page " + page.name)
 245                 page.delete('No longer published')
 246         for pagename, text in genp.items():
 247             logging.info("Checking " + pagename)
 248             if not pagename in existingpages:
 249                 logging.info("Creating page " + pagename)
 250                 try:
 251                     newpage = site.Pages[pagename]
 252                     newpage.save(text, summary='Auto-created')
 253                 except:
 254                     logging.warn("...FAILED to create page")
 255
 256     # Purge server cache to ensure counts are up to date
 257     site.pages['Repository Maintenance'].purge()
 258
 259 def delete_disabled_builds(apps, apkcache, repodirs):
 260     """Delete disabled build outputs.
 261
 262     :param apps: list of all applications, as per metadata.read_metadata
 263     :param apkcache: current apk cache information
 264     :param repodirs: the repo directories to process
 265     """
 266     for app in apps:
 267         for build in app['builds']:
 268             if 'disable' in build:
 269                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 270                 for repodir in repodirs:
 271                     apkpath = os.path.join(repodir, apkfilename)
 272                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 273                     for name in [apkpath, srcpath]:
 274                         if os.path.exists(name):
 275                             logging.info("Deleting disabled build output " + apkfilename)
 276                             os.remove(name)
 277                 if apkfilename in apkcache:
 278                     del apkcache[apkfilename]
 279
 280 def resize_icon(iconpath, density):
 281
 282     if not os.path.isfile(iconpath):
 283         return
 284
 285     try:
 286         im = Image.open(iconpath)
 287         size = dpi_to_px(density)
 288
 289         if any(length > size for length in im.size):
 290             oldsize = im.size
 291             im.thumbnail((size, size), Image.ANTIALIAS)
 292             logging.info("%s was too large at %s - new size is %s" % (
 293                 iconpath, oldsize, im.size))
 294             im.save(iconpath, "PNG")
 295
 296         else:
 297             logging.debug("%s is small enough: %s" % im.size)
 298
 299     except Exception,e:
 300         logging.warn("Failed resizing {0} - {1}".format(iconpath, e))
 301
 302 def resize_all_icons(repodirs):
 303     """Resize all icons that exceed the max size
 304
 305     :param repodirs: the repo directories to process
 306     """
 307     for repodir in repodirs:
 308         for density in get_densities():
 309             icon_dir = get_icon_dir(repodir, density)
 310             icon_glob = os.path.join(icon_dir, '*.png')
 311             for iconpath in glob.glob(icon_glob):
 312                 resize_icon(iconpath, density)
 313
 314 def scan_apks(apps, apkcache, repodir, knownapks):
 315     """Scan the apks in the given repo directory.
 316
 317     This also extracts the icons.
 318
 319     :param apps: list of all applications, as per metadata.read_metadata
 320     :param apkcache: current apk cache information
 321     :param repodir: repo directory to scan
 322     :param knownapks: known apks info
 323     :returns: (apks, cachechanged) where apks is a list of apk information,
 324               and cachechanged is True if the apkcache got changed.
 325     """
 326
 327     cachechanged = False
 328
 329     icon_dirs = get_icon_dirs(repodir)
 330     for icon_dir in icon_dirs:
 331         if os.path.exists(icon_dir):
 332             if options.clean:
 333                 shutil.rmtree(icon_dir)
 334                 os.makedirs(icon_dir)
 335         else:
 336             os.makedirs(icon_dir)
 337
 338     apks = []
 339     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 340     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 341     vername_pat = re.compile(".*versionName='([^']*)'.*")
 342     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 343     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 344     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 345     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 346     string_pat = re.compile(".*'([^']*)'.*")
 347     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 348
 349         apkfilename = apkfile[len(repodir) + 1:]
 350         if ' ' in apkfilename:
 351             logging.info("No spaces in APK filenames!")
 352             sys.exit(1)
 353
 354         if apkfilename in apkcache:
 355             logging.info("Reading " + apkfilename + " from cache")
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             if not options.quiet:
 361                 logging.info("Processing " + apkfilename)
 362             thisinfo = {}
 363             thisinfo['apkname'] = apkfilename
 364             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 365             if os.path.exists(os.path.join(repodir, srcfilename)):
 366                 thisinfo['srcname'] = srcfilename
 367             thisinfo['size'] = os.path.getsize(apkfile)
 368             thisinfo['permissions'] = []
 369             thisinfo['features'] = []
 370             thisinfo['icons_src'] = {}
 371             thisinfo['icons'] = {}
 372             p = FDroidPopen([os.path.join(config['sdk_path'],
 373                         'build-tools', config['build_tools'], 'aapt'),
 374                     'dump', 'badging', apkfile])
 375             if p.returncode != 0:
 376                 logging.critical("Failed to get apk information")
 377                 sys.exit(1)
 378             for line in p.stdout.splitlines():
 379                 if line.startswith("package:"):
 380                     try:
 381                         thisinfo['id'] = re.match(name_pat, line).group(1)
 382                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 383                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 384                     except Exception, e:
 385                         logging.info("Package matching failed: " + str(e))
 386                         logging.info("Line was: " + line)
 387                         sys.exit(1)
 388                 elif line.startswith("application:"):
 389                     thisinfo['name'] = re.match(label_pat, line).group(1)
 390                     # Keep path to non-dpi icon in case we need it
 391                     match = re.match(icon_pat_nodpi, line)
 392                     if match:
 393                         thisinfo['icons_src']['-1'] = match.group(1)
 394                 elif line.startswith("launchable-activity:"):
 395                     # Only use launchable-activity as fallback to application
 396                     if '-1' not in thisinfo['icons_src']:
 397                         match = re.match(icon_pat_nodpi, line)
 398                         if match:
 399                             thisinfo['icons_src']['-1'] = match.group(1)
 400                 elif line.startswith("application-icon-"):
 401                     match = re.match(icon_pat, line)
 402                     if match:
 403                         density = match.group(1)
 404                         path = match.group(2)
 405                         thisinfo['icons_src'][density] = path
 406                 elif line.startswith("sdkVersion:"):
 407                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 408                 elif line.startswith("native-code:"):
 409                     thisinfo['nativecode'] = []
 410                     for arch in line[13:].split(' '):
 411                         thisinfo['nativecode'].append(arch[1:-1])
 412                 elif line.startswith("uses-permission:"):
 413                     perm = re.match(string_pat, line).group(1)
 414                     if perm.startswith("android.permission."):
 415                         perm = perm[19:]
 416                     thisinfo['permissions'].append(perm)
 417                 elif line.startswith("uses-feature:"):
 418                     perm = re.match(string_pat, line).group(1)
 419                     #Filter out this, it's only added with the latest SDK tools and
 420                     #causes problems for lots of apps.
 421                     if (perm != "android.hardware.screen.portrait" and
 422                         perm != "android.hardware.screen.landscape"):
 423                         if perm.startswith("android.feature."):
 424                             perm = perm[16:]
 425                         thisinfo['features'].append(perm)
 426
 427             if not 'sdkversion' in thisinfo:
 428                 logging.warn("no SDK version information found")
 429                 thisinfo['sdkversion'] = 0
 430
 431             # Check for debuggable apks...
 432             if common.isApkDebuggable(apkfile, config):
 433                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 434
 435             # Calculate the sha256...
 436             sha = hashlib.sha256()
 437             with open(apkfile, 'rb') as f:
 438                 while True:
 439                     t = f.read(1024)
 440                     if len(t) == 0:
 441                         break
 442                     sha.update(t)
 443                 thisinfo['sha256'] = sha.hexdigest()
 444
 445             # Get the signature (or md5 of, to be precise)...
 446             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 447             if not os.path.exists(getsig_dir + "/getsig.class"):
 448                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 449                 sys.exit(1)
 450             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 451                         'getsig', os.path.join(os.getcwd(), apkfile)])
 452             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 453                 logging.critical("Failed to get apk signature")
 454                 sys.exit(1)
 455             thisinfo['sig'] = p.stdout[7:].strip()
 456
 457             apk = zipfile.ZipFile(apkfile, 'r')
 458
 459             iconfilename = "%s.%s.png" % (
 460                     thisinfo['id'],
 461                     thisinfo['versioncode'])
 462
 463             # Extract the icon file...
 464             densities = get_densities()
 465             empty_densities = []
 466             for density in densities:
 467                 if density not in thisinfo['icons_src']:
 468                     empty_densities.append(density)
 469                     continue
 470                 iconsrc = thisinfo['icons_src'][density]
 471                 icon_dir = get_icon_dir(repodir, density)
 472                 icondest = os.path.join(icon_dir, iconfilename)
 473
 474                 try:
 475                     iconfile = open(icondest, 'wb')
 476                     iconfile.write(apk.read(iconsrc))
 477                     iconfile.close()
 478                     thisinfo['icons'][density] = iconfilename
 479
 480                 except:
 481                     logging.warn("Error retrieving icon file")
 482                     del thisinfo['icons'][density]
 483                     del thisinfo['icons_src'][density]
 484                     empty_densities.append(density)
 485
 486             if '-1' in thisinfo['icons_src']:
 487                 iconsrc = thisinfo['icons_src']['-1']
 488                 iconpath = os.path.join(
 489                         get_icon_dir(repodir, None), iconfilename)
 490                 iconfile = open(iconpath, 'wb')
 491                 iconfile.write(apk.read(iconsrc))
 492                 iconfile.close()
 493                 try:
 494                     im = Image.open(iconpath)
 495                     dpi = px_to_dpi(im.size[0])
 496                     for density in densities:
 497                         if density in thisinfo['icons']:
 498                             break
 499                         if density == densities[-1] or dpi >= int(density):
 500                             thisinfo['icons'][density] = iconfilename
 501                             shutil.move(iconpath,
 502                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 503                             empty_densities.remove(density)
 504                             break
 505                 except Exception,e:
 506                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 507
 508             if thisinfo['icons']:
 509                 thisinfo['icon'] = iconfilename
 510
 511             apk.close()
 512
 513             # First try resizing down to not lose quality
 514             last_density = None
 515             for density in densities:
 516                 if density not in empty_densities:
 517                     last_density = density
 518                     continue
 519                 if last_density is None:
 520                     continue
 521                 logging.info("Density %s not available, resizing down from %s" % (
 522                         density, last_density))
 523
 524                 last_iconpath = os.path.join(
 525                         get_icon_dir(repodir, last_density), iconfilename)
 526                 iconpath = os.path.join(
 527                         get_icon_dir(repodir, density), iconfilename)
 528                 try:
 529                     im = Image.open(last_iconpath)
 530                 except:
 531                     logging.warn("Invalid image file at %s" % last_iconpath)
 532                     continue
 533
 534                 size = dpi_to_px(density)
 535
 536                 im.thumbnail((size, size), Image.ANTIALIAS)
 537                 im.save(iconpath, "PNG")
 538                 empty_densities.remove(density)
 539
 540             # Then just copy from the highest resolution available
 541             last_density = None
 542             for density in reversed(densities):
 543                 if density not in empty_densities:
 544                     last_density = density
 545                     continue
 546                 if last_density is None:
 547                     continue
 548                 logging.info("Density %s not available, copying from lower density %s" % (
 549                         density, last_density))
 550
 551                 shutil.copyfile(
 552                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 553                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 554
 555                 empty_densities.remove(density)
 556
 557             for density in densities:
 558                 icon_dir = get_icon_dir(repodir, density)
 559                 icondest = os.path.join(icon_dir, iconfilename)
 560                 resize_icon(icondest, density)
 561
 562             # Copy from icons-mdpi to icons since mdpi is the baseline density
 563             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 564             if os.path.isfile(baseline):
 565                 shutil.copyfile(baseline,
 566                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 567
 568             # Record in known apks, getting the added date at the same time..
 569             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 570             if added:
 571                 thisinfo['added'] = added
 572
 573             apkcache[apkfilename] = thisinfo
 574             cachechanged = True
 575
 576         apks.append(thisinfo)
 577
 578     return apks, cachechanged
 579
 580
 581 repo_pubkey_fingerprint = None
 582
 583 def make_index(apps, apks, repodir, archive, categories):
 584     """Make a repo index.
 585
 586     :param apps: fully populated apps list
 587     :param apks: full populated apks list
 588     :param repodir: the repo directory
 589     :param archive: True if this is the archive repo, False if it's the
 590                     main one.
 591     :param categories: list of categories
 592     """
 593
 594     doc = Document()
 595
 596     def addElement(name, value, doc, parent):
 597         el = doc.createElement(name)
 598         el.appendChild(doc.createTextNode(value))
 599         parent.appendChild(el)
 600     def addElementCDATA(name, value, doc, parent):
 601         el = doc.createElement(name)
 602         el.appendChild(doc.createCDATASection(value))
 603         parent.appendChild(el)
 604
 605     root = doc.createElement("fdroid")
 606     doc.appendChild(root)
 607
 608     repoel = doc.createElement("repo")
 609
 610     if archive:
 611         repoel.setAttribute("name", config['archive_name'])
 612         if config['repo_maxage'] != 0:
 613             repoel.setAttribute("maxage", str(config['repo_maxage']))
 614         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 615         repoel.setAttribute("url", config['archive_url'])
 616         addElement('description', config['archive_description'], doc, repoel)
 617
 618     else:
 619         repoel.setAttribute("name", config['repo_name'])
 620         if config['repo_maxage'] != 0:
 621             repoel.setAttribute("maxage", str(config['repo_maxage']))
 622         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 623         repoel.setAttribute("url", config['repo_url'])
 624         addElement('description', config['repo_description'], doc, repoel)
 625
 626     repoel.setAttribute("version", "11")
 627     repoel.setAttribute("timestamp", str(int(time.time())))
 628
 629     if config['repo_keyalias']:
 630
 631         # Generate a certificate fingerprint the same way keytool does it
 632         # (but with slightly different formatting)
 633         def cert_fingerprint(data):
 634             digest = hashlib.sha1(data).digest()
 635             ret = []
 636             for i in range(4):
 637                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 638             return " ".join(ret)
 639
 640         def extract_pubkey():
 641             p = FDroidPopen(['keytool', '-exportcert',
 642                                   '-alias', config['repo_keyalias'],
 643                                   '-keystore', config['keystore'],
 644                                   '-storepass', config['keystorepass']])
 645             if p.returncode != 0:
 646                 logging.critical("Failed to get repo pubkey")
 647                 sys.exit(1)
 648             global repo_pubkey_fingerprint
 649             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 650             return "".join("%02x" % ord(b) for b in p.stdout)
 651
 652         repoel.setAttribute("pubkey", extract_pubkey())
 653
 654     root.appendChild(repoel)
 655
 656     for app in apps:
 657
 658         if app['Disabled'] is not None:
 659             continue
 660
 661         # Get a list of the apks for this app...
 662         apklist = []
 663         for apk in apks:
 664             if apk['id'] == app['id']:
 665                 apklist.append(apk)
 666
 667         if len(apklist) == 0:
 668             continue
 669
 670         apel = doc.createElement("application")
 671         apel.setAttribute("id", app['id'])
 672         root.appendChild(apel)
 673
 674         addElement('id', app['id'], doc, apel)
 675         if 'added' in app:
 676             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 677         if 'lastupdated' in app:
 678             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 679         addElement('name', app['Name'], doc, apel)
 680         addElement('summary', app['Summary'], doc, apel)
 681         if app['icon']:
 682             addElement('icon', app['icon'], doc, apel)
 683         def linkres(link):
 684             for app in apps:
 685                 if app['id'] == link:
 686                     return ("fdroid.app:" + link, app['Name'])
 687             raise MetaDataException("Cannot resolve app id " + link)
 688         addElement('desc',
 689                 metadata.description_html(app['Description'], linkres), doc, apel)
 690         addElement('license', app['License'], doc, apel)
 691         if 'Categories' in app:
 692             addElement('categories', ','.join(app["Categories"]), doc, apel)
 693             # We put the first (primary) category in LAST, which will have
 694             # the desired effect of making clients that only understand one
 695             # category see that one.
 696             addElement('category', app["Categories"][0], doc, apel)
 697         addElement('web', app['Web Site'], doc, apel)
 698         addElement('source', app['Source Code'], doc, apel)
 699         addElement('tracker', app['Issue Tracker'], doc, apel)
 700         if app['Donate']:
 701             addElement('donate', app['Donate'], doc, apel)
 702         if app['Bitcoin']:
 703             addElement('bitcoin', app['Bitcoin'], doc, apel)
 704         if app['Litecoin']:
 705             addElement('litecoin', app['Litecoin'], doc, apel)
 706         if app['Dogecoin']:
 707             addElement('dogecoin', app['Dogecoin'], doc, apel)
 708         if app['FlattrID']:
 709             addElement('flattr', app['FlattrID'], doc, apel)
 710
 711         # These elements actually refer to the current version (i.e. which
 712         # one is recommended. They are historically mis-named, and need
 713         # changing, but stay like this for now to support existing clients.
 714         addElement('marketversion', app['Current Version'], doc, apel)
 715         addElement('marketvercode', app['Current Version Code'], doc, apel)
 716
 717         if app['AntiFeatures']:
 718             af = app['AntiFeatures'].split(',')
 719             # TODO: Temporarily not including UpstreamNonFree in the index,
 720             # because current F-Droid clients do not understand it, and also
 721             # look ugly when they encounter an unknown antifeature. This
 722             # filtering can be removed in time...
 723             if 'UpstreamNonFree' in af:
 724                 af.remove('UpstreamNonFree')
 725             if af:
 726                 addElement('antifeatures', ','.join(af), doc, apel)
 727         if app['Provides']:
 728             pv = app['Provides'].split(',')
 729             addElement('provides', ','.join(pv), doc, apel)
 730         if app['Requires Root']:
 731             addElement('requirements', 'root', doc, apel)
 732
 733         # Sort the apk list into version order, just so the web site
 734         # doesn't have to do any work by default...
 735         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 736
 737         # Check for duplicates - they will make the client unhappy...
 738         for i in range(len(apklist) - 1):
 739             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 740                 logging.critical("duplicate versions: '%s' - '%s'" % (
 741                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 742                 sys.exit(1)
 743
 744         for apk in apklist:
 745             apkel = doc.createElement("package")
 746             apel.appendChild(apkel)
 747             addElement('version', apk['version'], doc, apkel)
 748             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 749             addElement('apkname', apk['apkname'], doc, apkel)
 750             if 'srcname' in apk:
 751                 addElement('srcname', apk['srcname'], doc, apkel)
 752             for hash_type in ['sha256']:
 753                 if not hash_type in apk:
 754                     continue
 755                 hashel = doc.createElement("hash")
 756                 hashel.setAttribute("type", hash_type)
 757                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 758                 apkel.appendChild(hashel)
 759             addElement('sig', apk['sig'], doc, apkel)
 760             addElement('size', str(apk['size']), doc, apkel)
 761             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 762             if 'added' in apk:
 763                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 764             if app['Requires Root']:
 765                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 766                     apk['permissions'].append('ACCESS_SUPERUSER')
 767
 768             if len(apk['permissions']) > 0:
 769                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 770             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 771                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 772             if len(apk['features']) > 0:
 773                 addElement('features', ','.join(apk['features']), doc, apkel)
 774
 775     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 776     if options.pretty:
 777         output = doc.toprettyxml()
 778     else:
 779         output = doc.toxml()
 780     of.write(output)
 781     of.close()
 782
 783     if config['repo_keyalias'] is not None:
 784
 785         if not options.quiet:
 786             logging.info("Creating signed index.")
 787             logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint)
 788
 789         #Create a jar of the index...
 790         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 791         if p.returncode != 0:
 792             logging.critical("Failed to create jar file")
 793             sys.exit(1)
 794
 795         # Sign the index...
 796         p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
 797             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 798             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 799             os.path.join(repodir, 'index.jar') , config['repo_keyalias']])
 800         if p.returncode != 0:
 801             logging.info("Failed to sign index")
 802             sys.exit(1)
 803
 804     # Copy the repo icon into the repo directory...
 805     icon_dir = os.path.join(repodir ,'icons')
 806     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 807     shutil.copyfile(config['repo_icon'], iconfilename)
 808
 809     # Write a category list in the repo to allow quick access...
 810     catdata = ''
 811     for cat in categories:
 812         catdata += cat + '\n'
 813     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 814     f.write(catdata)
 815     f.close()
 816
 817
 818
 819 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 820
 821     for app in apps:
 822
 823         # Get a list of the apks for this app...
 824         apklist = []
 825         for apk in apks:
 826             if apk['id'] == app['id']:
 827                 apklist.append(apk)
 828
 829         # Sort the apk list into version order...
 830         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 831
 832         if app['Archive Policy']:
 833             keepversions = int(app['Archive Policy'][:-9])
 834         else:
 835             keepversions = defaultkeepversions
 836
 837         if len(apklist) > keepversions:
 838             for apk in apklist[keepversions:]:
 839                 logging.info("Moving " + apk['apkname'] + " to archive")
 840                 shutil.move(os.path.join(repodir, apk['apkname']),
 841                     os.path.join(archivedir, apk['apkname']))
 842                 if 'srcname' in apk:
 843                     shutil.move(os.path.join(repodir, apk['srcname']),
 844                         os.path.join(archivedir, apk['srcname']))
 845                 archapks.append(apk)
 846                 apks.remove(apk)
 847
 848
 849 config = None
 850 options = None
 851
 852 def main():
 853
 854     global config, options
 855
 856     # Parse command line...
 857     parser = OptionParser()
 858     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 859                       help="Create skeleton metadata files that are missing")
 860     parser.add_option("-v", "--verbose", action="store_true", default=False,
 861                       help="Spew out even more information than normal")
 862     parser.add_option("-q", "--quiet", action="store_true", default=False,
 863                       help="No output, except for warnings and errors")
 864     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 865                       help="Report on build data status")
 866     parser.add_option("-i", "--interactive", default=False, action="store_true",
 867                       help="Interactively ask about things that need updating.")
 868     parser.add_option("-I", "--icons", action="store_true", default=False,
 869                       help="Resize all the icons exceeding the max pixel size and exit")
 870     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 871                       help="Specify editor to use in interactive mode. Default "+
 872                           "is /etc/alternatives/editor")
 873     parser.add_option("-w", "--wiki", default=False, action="store_true",
 874                       help="Update the wiki")
 875     parser.add_option("", "--pretty", action="store_true", default=False,
 876                       help="Produce human-readable index.xml")
 877     parser.add_option("--clean", action="store_true", default=False,
 878                       help="Clean update - don't uses caches, reprocess all apks")
 879     (options, args) = parser.parse_args()
 880
 881     config = common.read_config(options)
 882
 883     repodirs = ['repo']
 884     if config['archive_older'] != 0:
 885         repodirs.append('archive')
 886         if not os.path.exists('archive'):
 887             os.mkdir('archive')
 888
 889     if options.icons:
 890         resize_all_icons(repodirs)
 891         sys.exit(0)
 892
 893     # Get all apps...
 894     apps = metadata.read_metadata()
 895
 896     # Generate a list of categories...
 897     categories = set()
 898     for app in apps:
 899         categories.update(app['Categories'])
 900
 901     # Read known apks data (will be updated and written back when we've finished)
 902     knownapks = common.KnownApks()
 903
 904     # Gather information about all the apk files in the repo directory, using
 905     # cached data if possible.
 906     apkcachefile = os.path.join('tmp', 'apkcache')
 907     if not options.clean and os.path.exists(apkcachefile):
 908         with open(apkcachefile, 'rb') as cf:
 909             apkcache = pickle.load(cf)
 910     else:
 911         apkcache = {}
 912     cachechanged = False
 913
 914     delete_disabled_builds(apps, apkcache, repodirs)
 915
 916     # Scan all apks in the main repo
 917     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 918     if cc:
 919         cachechanged = True
 920
 921     # Scan the archive repo for apks as well
 922     if len(repodirs) > 1:
 923         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 924         if cc:
 925             cachechanged = True
 926     else:
 927         archapks = []
 928
 929     # Some information from the apks needs to be applied up to the application
 930     # level. When doing this, we use the info from the most recent version's apk.
 931     # We deal with figuring out when the app was added and last updated at the
 932     # same time.
 933     for app in apps:
 934         bestver = 0
 935         added = None
 936         lastupdated = None
 937         for apk in apks + archapks:
 938             if apk['id'] == app['id']:
 939                 if apk['versioncode'] > bestver:
 940                     bestver = apk['versioncode']
 941                     bestapk = apk
 942
 943                 if 'added' in apk:
 944                     if not added or apk['added'] < added:
 945                         added = apk['added']
 946                     if not lastupdated or apk['added'] > lastupdated:
 947                         lastupdated = apk['added']
 948
 949         if added:
 950             app['added'] = added
 951         else:
 952             logging.warn("Don't know when " + app['id'] + " was added")
 953         if lastupdated:
 954             app['lastupdated'] = lastupdated
 955         else:
 956             logging.warn("Don't know when " + app['id'] + " was last updated")
 957
 958         if bestver == 0:
 959             if app['Name'] is None:
 960                 app['Name'] = app['id']
 961             app['icon'] = None
 962             logging.warn("Application " + app['id'] + " has no packages")
 963         else:
 964             if app['Name'] is None:
 965                 app['Name'] = bestapk['name']
 966             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 967
 968     # Sort the app list by name, then the web site doesn't have to by default.
 969     # (we had to wait until we'd scanned the apks to do this, because mostly the
 970     # name comes from there!)
 971     apps = sorted(apps, key=lambda app: app['Name'].upper())
 972
 973     # Generate warnings for apk's with no metadata (or create skeleton
 974     # metadata files, if requested on the command line)
 975     for apk in apks:
 976         found = False
 977         for app in apps:
 978             if app['id'] == apk['id']:
 979                 found = True
 980                 break
 981         if not found:
 982             if options.createmeta:
 983                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 984                 f.write("License:Unknown\n")
 985                 f.write("Web Site:\n")
 986                 f.write("Source Code:\n")
 987                 f.write("Issue Tracker:\n")
 988                 f.write("Summary:" + apk['name'] + "\n")
 989                 f.write("Description:\n")
 990                 f.write(apk['name'] + "\n")
 991                 f.write(".\n")
 992                 f.close()
 993                 logging.info("Generated skeleton metadata for " + apk['id'])
 994             else:
 995                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata")
 996                 logging.info("       " + apk['name'] + " - " + apk['version'])
 997
 998     if len(repodirs) > 1:
 999         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1000
1001     # Make the index for the main repo...
1002     make_index(apps, apks, repodirs[0], False, categories)
1003
1004     # If there's an archive repo,  make the index for it. We already scanned it
1005     # earlier on.
1006     if len(repodirs) > 1:
1007         make_index(apps, archapks, repodirs[1], True, categories)
1008
1009     if config['update_stats']:
1010
1011         # Update known apks info...
1012         knownapks.writeifchanged()
1013
1014         # Generate latest apps data for widget
1015         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1016             data = ''
1017             for line in file(os.path.join('stats', 'latestapps.txt')):
1018                 appid = line.rstrip()
1019                 data += appid + "\t"
1020                 for app in apps:
1021                     if app['id'] == appid:
1022                         data += app['Name'] + "\t"
1023                         if app['icon'] is not None:
1024                             data += app['icon'] + "\t"
1025                         data += app['License'] + "\n"
1026                         break
1027             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1028             f.write(data)
1029             f.close()
1030
1031     if cachechanged:
1032         with open(apkcachefile, 'wb') as cf:
1033             pickle.dump(apkcache, cf)
1034
1035     # Update the wiki...
1036     if options.wiki:
1037         update_wiki(apps, apks + archapks)
1038
1039     logging.info("Finished.")
1040
1041 if __name__ == "__main__":
1042     main()
1043