fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         # Sort with most recent first...
 145         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 146
 147         wikidata += "=Versions=\n"
 148         if len(apklist) == 0:
 149             wikidata += "We currently have no versions of this app available."
 150         elif not gotcurrentver:
 151             wikidata += "We don't have the current version of this app."
 152         else:
 153             wikidata += "We have the current version of this app."
 154         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 155         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 156         if len(app['No Source Since']) > 0:
 157             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 158         if len(app['Current Version']) > 0:
 159             wikidata += "The current (recommended) version is " + app['Current Version']
 160             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 161         validapks = 0
 162         for apk in apklist:
 163             wikidata += "==" + apk['version'] + "==\n"
 164
 165             if 'buildproblem' in apk:
 166                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 167             else:
 168                 validapks += 1
 169                 wikidata += "This version is built and signed by "
 170                 if 'srcname' in apk:
 171                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 172                 else:
 173                     wikidata += "the original developer.\n\n"
 174             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 175
 176         wikidata += '\n[[Category:' + wikicat + ']]\n'
 177         if len(app['No Source Since']) > 0:
 178             wikidata += '\n[[Category:Apps missing source code]]\n'
 179         if validapks == 0 and not app['Disabled']:
 180             wikidata += '\n[[Category:Apps with no packages]]\n'
 181         if cantupdate and not app['Disabled']:
 182             wikidata += "\n[[Category:Apps we can't update]]\n"
 183         if buildfails and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps with failing builds]]\n"
 185         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 186             wikidata += '\n[[Category:Apps to Update]]\n'
 187         if app['Disabled']:
 188             wikidata += '\n[[Category:Apps that are disabled]]\n'
 189         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 190             wikidata += '\n[[Category:Apps with no update check]]\n'
 191         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 192             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 193
 194         # We can't have underscores in the page name, even if they're in
 195         # the package ID, because MediaWiki messes with them...
 196         pagename = app['id'].replace('_', ' ')
 197
 198         # Drop a trailing newline, because mediawiki is going to drop it anyway
 199         # and it we don't we'll think the page has changed when it hasn't...
 200         if wikidata.endswith('\n'):
 201             wikidata = wikidata[:-1]
 202
 203         generated_pages[pagename] = wikidata
 204
 205         # Make a redirect from the name to the ID too, unless there's
 206         # already an existing page with the name and it isn't a redirect.
 207         noclobber = False
 208         apppagename = app['Name'].replace('_', ' ')
 209         apppagename = apppagename.replace('{', '')
 210         apppagename = apppagename.replace('}', ' ')
 211         apppagename = apppagename.replace(':', ' ')
 212         # Drop double spaces caused mostly by replacing ':' above
 213         apppagename = apppagename.replace('  ', ' ')
 214         for expagename in site.allpages(prefix=apppagename,
 215                 filterredir='nonredirects', generator=False):
 216             if expagename == apppagename:
 217                 noclobber = True
 218         # Another reason not to make the redirect page is if the app name
 219         # is the same as it's ID, because that will overwrite the real page
 220         # with an redirect to itself! (Although it seems like an odd
 221         # scenario this happens a lot, e.g. where there is metadata but no
 222         # builds or binaries to extract a name from.
 223         if apppagename == pagename:
 224             noclobber = True
 225         if not noclobber:
 226             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 227
 228     for tcat, genp in [(wikicat, generated_pages),
 229             (wikiredircat, generated_redirects)]:
 230         catpages = site.Pages['Category:' + tcat]
 231         existingpages = []
 232         for page in catpages:
 233             existingpages.append(page.name)
 234             if page.name in genp:
 235                 pagetxt = page.edit()
 236                 if pagetxt != genp[page.name]:
 237                     logging.info("Updating modified page " + page.name)
 238                     page.save(genp[page.name], summary='Auto-updated')
 239                 else:
 240                     logging.info("Page " + page.name + " is unchanged")
 241             else:
 242                 logging.info("Deleting page " + page.name)
 243                 page.delete('No longer published')
 244         for pagename, text in genp.items():
 245             logging.info("Checking " + pagename)
 246             if not pagename in existingpages:
 247                 logging.info("Creating page " + pagename)
 248                 try:
 249                     newpage = site.Pages[pagename]
 250                     newpage.save(text, summary='Auto-created')
 251                 except:
 252                     logging.warn("...FAILED to create page")
 253
 254     # Purge server cache to ensure counts are up to date
 255     site.pages['Repository Maintenance'].purge()
 256
 257 def delete_disabled_builds(apps, apkcache, repodirs):
 258     """Delete disabled build outputs.
 259
 260     :param apps: list of all applications, as per metadata.read_metadata
 261     :param apkcache: current apk cache information
 262     :param repodirs: the repo directories to process
 263     """
 264     for app in apps:
 265         for build in app['builds']:
 266             if 'disable' in build:
 267                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 268                 for repodir in repodirs:
 269                     apkpath = os.path.join(repodir, apkfilename)
 270                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 271                     for name in [apkpath, srcpath]:
 272                         if os.path.exists(name):
 273                             logging.info("Deleting disabled build output " + apkfilename)
 274                             os.remove(name)
 275                 if apkfilename in apkcache:
 276                     del apkcache[apkfilename]
 277
 278 def resize_icon(iconpath, density):
 279
 280     if not os.path.isfile(iconpath):
 281         return
 282
 283     try:
 284         im = Image.open(iconpath)
 285         size = dpi_to_px(density)
 286
 287         if any(length > size for length in im.size):
 288             oldsize = im.size
 289             im.thumbnail((size, size), Image.ANTIALIAS)
 290             logging.info("%s was too large at %s - new size is %s" % (
 291                 iconpath, oldsize, im.size))
 292             im.save(iconpath, "PNG")
 293
 294         else:
 295             logging.debug("%s is small enough: %s" % im.size)
 296
 297     except Exception,e:
 298         logging.info("WARNING: Failed resizing {0} - {1}".format(iconpath, e))
 299
 300 def resize_all_icons(repodirs):
 301     """Resize all icons that exceed the max size
 302
 303     :param repodirs: the repo directories to process
 304     """
 305     for repodir in repodirs:
 306         for density in get_densities():
 307             icon_dir = get_icon_dir(repodir, density)
 308             icon_glob = os.path.join(icon_dir, '*.png')
 309             for iconpath in glob.glob(icon_glob):
 310                 resize_icon(iconpath, density)
 311
 312 def scan_apks(apps, apkcache, repodir, knownapks):
 313     """Scan the apks in the given repo directory.
 314
 315     This also extracts the icons.
 316
 317     :param apps: list of all applications, as per metadata.read_metadata
 318     :param apkcache: current apk cache information
 319     :param repodir: repo directory to scan
 320     :param knownapks: known apks info
 321     :returns: (apks, cachechanged) where apks is a list of apk information,
 322               and cachechanged is True if the apkcache got changed.
 323     """
 324
 325     cachechanged = False
 326
 327     icon_dirs = get_icon_dirs(repodir)
 328     for icon_dir in icon_dirs:
 329         if os.path.exists(icon_dir):
 330             if options.clean:
 331                 shutil.rmtree(icon_dir)
 332                 os.makedirs(icon_dir)
 333         else:
 334             os.makedirs(icon_dir)
 335
 336     apks = []
 337     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 338     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 339     vername_pat = re.compile(".*versionName='([^']*)'.*")
 340     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 341     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 342     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 343     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 344     string_pat = re.compile(".*'([^']*)'.*")
 345     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 346
 347         apkfilename = apkfile[len(repodir) + 1:]
 348         if ' ' in apkfilename:
 349             logging.info("No spaces in APK filenames!")
 350             sys.exit(1)
 351
 352         if apkfilename in apkcache:
 353             logging.info("Reading " + apkfilename + " from cache")
 354             thisinfo = apkcache[apkfilename]
 355
 356         else:
 357
 358             if not options.quiet:
 359                 logging.info("Processing " + apkfilename)
 360             thisinfo = {}
 361             thisinfo['apkname'] = apkfilename
 362             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 363             if os.path.exists(os.path.join(repodir, srcfilename)):
 364                 thisinfo['srcname'] = srcfilename
 365             thisinfo['size'] = os.path.getsize(apkfile)
 366             thisinfo['permissions'] = []
 367             thisinfo['features'] = []
 368             thisinfo['icons_src'] = {}
 369             thisinfo['icons'] = {}
 370             p = FDroidPopen([os.path.join(config['sdk_path'],
 371                         'build-tools', config['build_tools'], 'aapt'),
 372                     'dump', 'badging', apkfile])
 373             if p.returncode != 0:
 374                 logging.critical("Failed to get apk information")
 375                 sys.exit(1)
 376             for line in p.stdout.splitlines():
 377                 if line.startswith("package:"):
 378                     try:
 379                         thisinfo['id'] = re.match(name_pat, line).group(1)
 380                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 381                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 382                     except Exception, e:
 383                         logging.info("Package matching failed: " + str(e))
 384                         logging.info("Line was: " + line)
 385                         sys.exit(1)
 386                 elif line.startswith("application:"):
 387                     thisinfo['name'] = re.match(label_pat, line).group(1)
 388                     # Keep path to non-dpi icon in case we need it
 389                     match = re.match(icon_pat_nodpi, line)
 390                     if match:
 391                         thisinfo['icons_src']['-1'] = match.group(1)
 392                 elif line.startswith("launchable-activity:"):
 393                     # Only use launchable-activity as fallback to application
 394                     if '-1' not in thisinfo['icons_src']:
 395                         match = re.match(icon_pat_nodpi, line)
 396                         if match:
 397                             thisinfo['icons_src']['-1'] = match.group(1)
 398                 elif line.startswith("application-icon-"):
 399                     match = re.match(icon_pat, line)
 400                     if match:
 401                         density = match.group(1)
 402                         path = match.group(2)
 403                         thisinfo['icons_src'][density] = path
 404                 elif line.startswith("sdkVersion:"):
 405                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 406                 elif line.startswith("native-code:"):
 407                     thisinfo['nativecode'] = []
 408                     for arch in line[13:].split(' '):
 409                         thisinfo['nativecode'].append(arch[1:-1])
 410                 elif line.startswith("uses-permission:"):
 411                     perm = re.match(string_pat, line).group(1)
 412                     if perm.startswith("android.permission."):
 413                         perm = perm[19:]
 414                     thisinfo['permissions'].append(perm)
 415                 elif line.startswith("uses-feature:"):
 416                     perm = re.match(string_pat, line).group(1)
 417                     #Filter out this, it's only added with the latest SDK tools and
 418                     #causes problems for lots of apps.
 419                     if (perm != "android.hardware.screen.portrait" and
 420                         perm != "android.hardware.screen.landscape"):
 421                         if perm.startswith("android.feature."):
 422                             perm = perm[16:]
 423                         thisinfo['features'].append(perm)
 424
 425             if not 'sdkversion' in thisinfo:
 426                 logging.info("  WARNING: no SDK version information found")
 427                 thisinfo['sdkversion'] = 0
 428
 429             # Check for debuggable apks...
 430             if common.isApkDebuggable(apkfile, config):
 431                 logging.info("WARNING: {0} is debuggable... {1}".format(apkfile, line))
 432
 433             # Calculate the sha256...
 434             sha = hashlib.sha256()
 435             with open(apkfile, 'rb') as f:
 436                 while True:
 437                     t = f.read(1024)
 438                     if len(t) == 0:
 439                         break
 440                     sha.update(t)
 441                 thisinfo['sha256'] = sha.hexdigest()
 442
 443             # Get the signature (or md5 of, to be precise)...
 444             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 445             if not os.path.exists(getsig_dir + "/getsig.class"):
 446                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 447                 sys.exit(1)
 448             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 449                         'getsig', os.path.join(os.getcwd(), apkfile)])
 450             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 451                 logging.critical("Failed to get apk signature")
 452                 sys.exit(1)
 453             thisinfo['sig'] = p.stdout[7:].strip()
 454
 455             apk = zipfile.ZipFile(apkfile, 'r')
 456
 457             iconfilename = "%s.%s.png" % (
 458                     thisinfo['id'],
 459                     thisinfo['versioncode'])
 460
 461             # Extract the icon file...
 462             densities = get_densities()
 463             empty_densities = []
 464             for density in densities:
 465                 if density not in thisinfo['icons_src']:
 466                     empty_densities.append(density)
 467                     continue
 468                 iconsrc = thisinfo['icons_src'][density]
 469                 icon_dir = get_icon_dir(repodir, density)
 470                 icondest = os.path.join(icon_dir, iconfilename)
 471
 472                 try:
 473                     iconfile = open(icondest, 'wb')
 474                     iconfile.write(apk.read(iconsrc))
 475                     iconfile.close()
 476                     thisinfo['icons'][density] = iconfilename
 477
 478                 except:
 479                     logging.info("WARNING: Error retrieving icon file")
 480                     del thisinfo['icons'][density]
 481                     del thisinfo['icons_src'][density]
 482                     empty_densities.append(density)
 483
 484             if '-1' in thisinfo['icons_src']:
 485                 iconsrc = thisinfo['icons_src']['-1']
 486                 iconpath = os.path.join(
 487                         get_icon_dir(repodir, None), iconfilename)
 488                 iconfile = open(iconpath, 'wb')
 489                 iconfile.write(apk.read(iconsrc))
 490                 iconfile.close()
 491                 try:
 492                     im = Image.open(iconpath)
 493                     dpi = px_to_dpi(im.size[0])
 494                     for density in densities:
 495                         if density in thisinfo['icons']:
 496                             break
 497                         if density == densities[-1] or dpi >= int(density):
 498                             thisinfo['icons'][density] = iconfilename
 499                             shutil.move(iconpath,
 500                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 501                             empty_densities.remove(density)
 502                             break
 503                 except Exception,e:
 504                     logging.info("WARNING: Failed reading {0} - {1}".format(iconpath, e))
 505
 506             if thisinfo['icons']:
 507                 thisinfo['icon'] = iconfilename
 508
 509             apk.close()
 510
 511             # First try resizing down to not lose quality
 512             last_density = None
 513             for density in densities:
 514                 if density not in empty_densities:
 515                     last_density = density
 516                     continue
 517                 if last_density is None:
 518                     continue
 519                 logging.info("Density %s not available, resizing down from %s" % (
 520                         density, last_density))
 521
 522                 last_iconpath = os.path.join(
 523                         get_icon_dir(repodir, last_density), iconfilename)
 524                 iconpath = os.path.join(
 525                         get_icon_dir(repodir, density), iconfilename)
 526                 try:
 527                     im = Image.open(last_iconpath)
 528                 except:
 529                     logging.info("WARNING: Invalid image file at %s" % last_iconpath)
 530                     continue
 531
 532                 size = dpi_to_px(density)
 533
 534                 im.thumbnail((size, size), Image.ANTIALIAS)
 535                 im.save(iconpath, "PNG")
 536                 empty_densities.remove(density)
 537
 538             # Then just copy from the highest resolution available
 539             last_density = None
 540             for density in reversed(densities):
 541                 if density not in empty_densities:
 542                     last_density = density
 543                     continue
 544                 if last_density is None:
 545                     continue
 546                 logging.info("Density %s not available, copying from lower density %s" % (
 547                         density, last_density))
 548
 549                 shutil.copyfile(
 550                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 551                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 552
 553                 empty_densities.remove(density)
 554
 555             for density in densities:
 556                 icon_dir = get_icon_dir(repodir, density)
 557                 icondest = os.path.join(icon_dir, iconfilename)
 558                 resize_icon(icondest, density)
 559
 560             # Copy from icons-mdpi to icons since mdpi is the baseline density
 561             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 562             if os.path.isfile(baseline):
 563                 shutil.copyfile(baseline,
 564                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 565
 566             # Record in known apks, getting the added date at the same time..
 567             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 568             if added:
 569                 thisinfo['added'] = added
 570
 571             apkcache[apkfilename] = thisinfo
 572             cachechanged = True
 573
 574         apks.append(thisinfo)
 575
 576     return apks, cachechanged
 577
 578
 579 repo_pubkey_fingerprint = None
 580
 581 def make_index(apps, apks, repodir, archive, categories):
 582     """Make a repo index.
 583
 584     :param apps: fully populated apps list
 585     :param apks: full populated apks list
 586     :param repodir: the repo directory
 587     :param archive: True if this is the archive repo, False if it's the
 588                     main one.
 589     :param categories: list of categories
 590     """
 591
 592     doc = Document()
 593
 594     def addElement(name, value, doc, parent):
 595         el = doc.createElement(name)
 596         el.appendChild(doc.createTextNode(value))
 597         parent.appendChild(el)
 598     def addElementCDATA(name, value, doc, parent):
 599         el = doc.createElement(name)
 600         el.appendChild(doc.createCDATASection(value))
 601         parent.appendChild(el)
 602
 603     root = doc.createElement("fdroid")
 604     doc.appendChild(root)
 605
 606     repoel = doc.createElement("repo")
 607
 608     if archive:
 609         repoel.setAttribute("name", config['archive_name'])
 610         if config['repo_maxage'] != 0:
 611             repoel.setAttribute("maxage", str(config['repo_maxage']))
 612         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 613         repoel.setAttribute("url", config['archive_url'])
 614         addElement('description', config['archive_description'], doc, repoel)
 615
 616     else:
 617         repoel.setAttribute("name", config['repo_name'])
 618         if config['repo_maxage'] != 0:
 619             repoel.setAttribute("maxage", str(config['repo_maxage']))
 620         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 621         repoel.setAttribute("url", config['repo_url'])
 622         addElement('description', config['repo_description'], doc, repoel)
 623
 624     repoel.setAttribute("version", "11")
 625     repoel.setAttribute("timestamp", str(int(time.time())))
 626
 627     if config['repo_keyalias']:
 628
 629         # Generate a certificate fingerprint the same way keytool does it
 630         # (but with slightly different formatting)
 631         def cert_fingerprint(data):
 632             digest = hashlib.sha1(data).digest()
 633             ret = []
 634             for i in range(4):
 635                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 636             return " ".join(ret)
 637
 638         def extract_pubkey():
 639             p = FDroidPopen(['keytool', '-exportcert',
 640                                   '-alias', config['repo_keyalias'],
 641                                   '-keystore', config['keystore'],
 642                                   '-storepass', config['keystorepass']])
 643             if p.returncode != 0:
 644                 logging.critical("Failed to get repo pubkey")
 645                 sys.exit(1)
 646             global repo_pubkey_fingerprint
 647             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 648             return "".join("%02x" % ord(b) for b in p.stdout)
 649
 650         repoel.setAttribute("pubkey", extract_pubkey())
 651
 652     root.appendChild(repoel)
 653
 654     for app in apps:
 655
 656         if app['Disabled'] is not None:
 657             continue
 658
 659         # Get a list of the apks for this app...
 660         apklist = []
 661         for apk in apks:
 662             if apk['id'] == app['id']:
 663                 apklist.append(apk)
 664
 665         if len(apklist) == 0:
 666             continue
 667
 668         apel = doc.createElement("application")
 669         apel.setAttribute("id", app['id'])
 670         root.appendChild(apel)
 671
 672         addElement('id', app['id'], doc, apel)
 673         if 'added' in app:
 674             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 675         if 'lastupdated' in app:
 676             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 677         addElement('name', app['Name'], doc, apel)
 678         addElement('summary', app['Summary'], doc, apel)
 679         if app['icon']:
 680             addElement('icon', app['icon'], doc, apel)
 681         def linkres(link):
 682             for app in apps:
 683                 if app['id'] == link:
 684                     return ("fdroid.app:" + link, app['Name'])
 685             raise MetaDataException("Cannot resolve app id " + link)
 686         addElement('desc',
 687                 metadata.description_html(app['Description'], linkres), doc, apel)
 688         addElement('license', app['License'], doc, apel)
 689         if 'Categories' in app:
 690             appcategories = [c.strip() for c in app['Categories'].split(',')]
 691             addElement('categories', ','.join(appcategories), doc, apel)
 692             # We put the first (primary) category in LAST, which will have
 693             # the desired effect of making clients that only understand one
 694             # category see that one.
 695             addElement('category', appcategories[0], doc, apel)
 696         addElement('web', app['Web Site'], doc, apel)
 697         addElement('source', app['Source Code'], doc, apel)
 698         addElement('tracker', app['Issue Tracker'], doc, apel)
 699         if app['Donate']:
 700             addElement('donate', app['Donate'], doc, apel)
 701         if app['Bitcoin']:
 702             addElement('bitcoin', app['Bitcoin'], doc, apel)
 703         if app['Litecoin']:
 704             addElement('litecoin', app['Litecoin'], doc, apel)
 705         if app['Dogecoin']:
 706             addElement('dogecoin', app['Dogecoin'], doc, apel)
 707         if app['FlattrID']:
 708             addElement('flattr', app['FlattrID'], doc, apel)
 709
 710         # These elements actually refer to the current version (i.e. which
 711         # one is recommended. They are historically mis-named, and need
 712         # changing, but stay like this for now to support existing clients.
 713         addElement('marketversion', app['Current Version'], doc, apel)
 714         addElement('marketvercode', app['Current Version Code'], doc, apel)
 715
 716         if app['AntiFeatures']:
 717             af = app['AntiFeatures'].split(',')
 718             # TODO: Temporarily not including UpstreamNonFree in the index,
 719             # because current F-Droid clients do not understand it, and also
 720             # look ugly when they encounter an unknown antifeature. This
 721             # filtering can be removed in time...
 722             if 'UpstreamNonFree' in af:
 723                 af.remove('UpstreamNonFree')
 724             if af:
 725                 addElement('antifeatures', ','.join(af), doc, apel)
 726         if app['Provides']:
 727             pv = app['Provides'].split(',')
 728             addElement('provides', ','.join(pv), doc, apel)
 729         if app['Requires Root']:
 730             addElement('requirements', 'root', doc, apel)
 731
 732         # Sort the apk list into version order, just so the web site
 733         # doesn't have to do any work by default...
 734         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 735
 736         # Check for duplicates - they will make the client unhappy...
 737         for i in range(len(apklist) - 1):
 738             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 739                 logging.critical("duplicate versions: '%s' - '%s'" % (
 740                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 741                 sys.exit(1)
 742
 743         for apk in apklist:
 744             apkel = doc.createElement("package")
 745             apel.appendChild(apkel)
 746             addElement('version', apk['version'], doc, apkel)
 747             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 748             addElement('apkname', apk['apkname'], doc, apkel)
 749             if 'srcname' in apk:
 750                 addElement('srcname', apk['srcname'], doc, apkel)
 751             for hash_type in ['sha256']:
 752                 if not hash_type in apk:
 753                     continue
 754                 hashel = doc.createElement("hash")
 755                 hashel.setAttribute("type", hash_type)
 756                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 757                 apkel.appendChild(hashel)
 758             addElement('sig', apk['sig'], doc, apkel)
 759             addElement('size', str(apk['size']), doc, apkel)
 760             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 761             if 'added' in apk:
 762                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 763             if app['Requires Root']:
 764                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 765                     apk['permissions'].append('ACCESS_SUPERUSER')
 766
 767             if len(apk['permissions']) > 0:
 768                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 769             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 770                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 771             if len(apk['features']) > 0:
 772                 addElement('features', ','.join(apk['features']), doc, apkel)
 773
 774     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 775     if options.pretty:
 776         output = doc.toprettyxml()
 777     else:
 778         output = doc.toxml()
 779     of.write(output)
 780     of.close()
 781
 782     if config['repo_keyalias'] is not None:
 783
 784         if not options.quiet:
 785             logging.info("Creating signed index.")
 786             logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint)
 787
 788         #Create a jar of the index...
 789         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 790         if p.returncode != 0:
 791             logging.critical("Failed to create jar file")
 792             sys.exit(1)
 793
 794         # Sign the index...
 795         p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
 796             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 797             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 798             os.path.join(repodir, 'index.jar') , config['repo_keyalias']])
 799         if p.returncode != 0:
 800             logging.info("Failed to sign index")
 801             sys.exit(1)
 802
 803     # Copy the repo icon into the repo directory...
 804     icon_dir = os.path.join(repodir ,'icons')
 805     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 806     shutil.copyfile(config['repo_icon'], iconfilename)
 807
 808     # Write a category list in the repo to allow quick access...
 809     catdata = ''
 810     for cat in categories:
 811         catdata += cat + '\n'
 812     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 813     f.write(catdata)
 814     f.close()
 815
 816
 817
 818 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 819
 820     for app in apps:
 821
 822         # Get a list of the apks for this app...
 823         apklist = []
 824         for apk in apks:
 825             if apk['id'] == app['id']:
 826                 apklist.append(apk)
 827
 828         # Sort the apk list into version order...
 829         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 830
 831         if app['Archive Policy']:
 832             keepversions = int(app['Archive Policy'][:-9])
 833         else:
 834             keepversions = defaultkeepversions
 835
 836         if len(apklist) > keepversions:
 837             for apk in apklist[keepversions:]:
 838                 logging.info("Moving " + apk['apkname'] + " to archive")
 839                 shutil.move(os.path.join(repodir, apk['apkname']),
 840                     os.path.join(archivedir, apk['apkname']))
 841                 if 'srcname' in apk:
 842                     shutil.move(os.path.join(repodir, apk['srcname']),
 843                         os.path.join(archivedir, apk['srcname']))
 844                 archapks.append(apk)
 845                 apks.remove(apk)
 846
 847
 848 config = None
 849 options = None
 850
 851 def main():
 852
 853     global config, options
 854
 855     # Parse command line...
 856     parser = OptionParser()
 857     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 858                       help="Create skeleton metadata files that are missing")
 859     parser.add_option("-v", "--verbose", action="store_true", default=False,
 860                       help="Spew out even more information than normal")
 861     parser.add_option("-q", "--quiet", action="store_true", default=False,
 862                       help="No output, except for warnings and errors")
 863     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 864                       help="Report on build data status")
 865     parser.add_option("-i", "--interactive", default=False, action="store_true",
 866                       help="Interactively ask about things that need updating.")
 867     parser.add_option("-I", "--icons", action="store_true", default=False,
 868                       help="Resize all the icons exceeding the max pixel size and exit")
 869     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 870                       help="Specify editor to use in interactive mode. Default "+
 871                           "is /etc/alternatives/editor")
 872     parser.add_option("-w", "--wiki", default=False, action="store_true",
 873                       help="Update the wiki")
 874     parser.add_option("", "--pretty", action="store_true", default=False,
 875                       help="Produce human-readable index.xml")
 876     parser.add_option("--clean", action="store_true", default=False,
 877                       help="Clean update - don't uses caches, reprocess all apks")
 878     (options, args) = parser.parse_args()
 879
 880     config = common.read_config(options)
 881
 882     repodirs = ['repo']
 883     if config['archive_older'] != 0:
 884         repodirs.append('archive')
 885         if not os.path.exists('archive'):
 886             os.mkdir('archive')
 887
 888     if options.icons:
 889         resize_all_icons(repodirs)
 890         sys.exit(0)
 891
 892     # Get all apps...
 893     apps = metadata.read_metadata()
 894
 895     # Generate a list of categories...
 896     categories = []
 897     for app in apps:
 898         cats = app['Categories'].split(',')
 899         for cat in cats:
 900             if cat not in categories:
 901                 categories.append(cat)
 902
 903     # Read known apks data (will be updated and written back when we've finished)
 904     knownapks = common.KnownApks()
 905
 906     # Gather information about all the apk files in the repo directory, using
 907     # cached data if possible.
 908     apkcachefile = os.path.join('tmp', 'apkcache')
 909     if not options.clean and os.path.exists(apkcachefile):
 910         with open(apkcachefile, 'rb') as cf:
 911             apkcache = pickle.load(cf)
 912     else:
 913         apkcache = {}
 914     cachechanged = False
 915
 916     delete_disabled_builds(apps, apkcache, repodirs)
 917
 918     # Scan all apks in the main repo
 919     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 920     if cc:
 921         cachechanged = True
 922
 923     # Scan the archive repo for apks as well
 924     if len(repodirs) > 1:
 925         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 926         if cc:
 927             cachechanged = True
 928     else:
 929         archapks = []
 930
 931     # Some information from the apks needs to be applied up to the application
 932     # level. When doing this, we use the info from the most recent version's apk.
 933     # We deal with figuring out when the app was added and last updated at the
 934     # same time.
 935     for app in apps:
 936         bestver = 0
 937         added = None
 938         lastupdated = None
 939         for apk in apks + archapks:
 940             if apk['id'] == app['id']:
 941                 if apk['versioncode'] > bestver:
 942                     bestver = apk['versioncode']
 943                     bestapk = apk
 944
 945                 if 'added' in apk:
 946                     if not added or apk['added'] < added:
 947                         added = apk['added']
 948                     if not lastupdated or apk['added'] > lastupdated:
 949                         lastupdated = apk['added']
 950
 951         if added:
 952             app['added'] = added
 953         else:
 954             logging.info("WARNING: Don't know when " + app['id'] + " was added")
 955         if lastupdated:
 956             app['lastupdated'] = lastupdated
 957         else:
 958             logging.info("WARNING: Don't know when " + app['id'] + " was last updated")
 959
 960         if bestver == 0:
 961             if app['Name'] is None:
 962                 app['Name'] = app['id']
 963             app['icon'] = None
 964             logging.info("WARNING: Application " + app['id'] + " has no packages")
 965         else:
 966             if app['Name'] is None:
 967                 app['Name'] = bestapk['name']
 968             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 969
 970     # Sort the app list by name, then the web site doesn't have to by default.
 971     # (we had to wait until we'd scanned the apks to do this, because mostly the
 972     # name comes from there!)
 973     apps = sorted(apps, key=lambda app: app['Name'].upper())
 974
 975     # Generate warnings for apk's with no metadata (or create skeleton
 976     # metadata files, if requested on the command line)
 977     for apk in apks:
 978         found = False
 979         for app in apps:
 980             if app['id'] == apk['id']:
 981                 found = True
 982                 break
 983         if not found:
 984             if options.createmeta:
 985                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 986                 f.write("License:Unknown\n")
 987                 f.write("Web Site:\n")
 988                 f.write("Source Code:\n")
 989                 f.write("Issue Tracker:\n")
 990                 f.write("Summary:" + apk['name'] + "\n")
 991                 f.write("Description:\n")
 992                 f.write(apk['name'] + "\n")
 993                 f.write(".\n")
 994                 f.close()
 995                 logging.info("Generated skeleton metadata for " + apk['id'])
 996             else:
 997                 logging.info("WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata")
 998                 logging.info("       " + apk['name'] + " - " + apk['version'])
 999
1000     if len(repodirs) > 1:
1001         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1002
1003     # Make the index for the main repo...
1004     make_index(apps, apks, repodirs[0], False, categories)
1005
1006     # If there's an archive repo,  make the index for it. We already scanned it
1007     # earlier on.
1008     if len(repodirs) > 1:
1009         make_index(apps, archapks, repodirs[1], True, categories)
1010
1011     if config['update_stats']:
1012
1013         # Update known apks info...
1014         knownapks.writeifchanged()
1015
1016         # Generate latest apps data for widget
1017         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1018             data = ''
1019             for line in file(os.path.join('stats', 'latestapps.txt')):
1020                 appid = line.rstrip()
1021                 data += appid + "\t"
1022                 for app in apps:
1023                     if app['id'] == appid:
1024                         data += app['Name'] + "\t"
1025                         if app['icon'] is not None:
1026                             data += app['icon'] + "\t"
1027                         data += app['License'] + "\n"
1028                         break
1029             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1030             f.write(data)
1031             f.close()
1032
1033     if cachechanged:
1034         with open(apkcachefile, 'wb') as cf:
1035             pickle.dump(apkcache, cf)
1036
1037     # Update the wiki...
1038     if options.wiki:
1039         update_wiki(apps, apks + archapks)
1040
1041     logging.info("Finished.")
1042
1043 if __name__ == "__main__":
1044     main()
1045