fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         # Sort with most recent first...
 145         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 146
 147         wikidata += "=Versions=\n"
 148         if len(apklist) == 0:
 149             wikidata += "We currently have no versions of this app available."
 150         elif not gotcurrentver:
 151             wikidata += "We don't have the current version of this app."
 152         else:
 153             wikidata += "We have the current version of this app."
 154         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 155         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 156         if len(app['No Source Since']) > 0:
 157             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 158         if len(app['Current Version']) > 0:
 159             wikidata += "The current (recommended) version is " + app['Current Version']
 160             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 161         validapks = 0
 162         for apk in apklist:
 163             wikidata += "==" + apk['version'] + "==\n"
 164
 165             if 'buildproblem' in apk:
 166                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 167             else:
 168                 validapks += 1
 169                 wikidata += "This version is built and signed by "
 170                 if 'srcname' in apk:
 171                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 172                 else:
 173                     wikidata += "the original developer.\n\n"
 174             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 175
 176         wikidata += '\n[[Category:' + wikicat + ']]\n'
 177         if len(app['No Source Since']) > 0:
 178             wikidata += '\n[[Category:Apps missing source code]]\n'
 179         if validapks == 0 and not app['Disabled']:
 180             wikidata += '\n[[Category:Apps with no packages]]\n'
 181         if cantupdate and not app['Disabled']:
 182             wikidata += "\n[[Category:Apps we can't update]]\n"
 183         if buildfails and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps with failing builds]]\n"
 185         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 186             wikidata += '\n[[Category:Apps to Update]]\n'
 187         if app['Disabled']:
 188             wikidata += '\n[[Category:Apps that are disabled]]\n'
 189         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 190             wikidata += '\n[[Category:Apps with no update check]]\n'
 191         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 192             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 193
 194         # We can't have underscores in the page name, even if they're in
 195         # the package ID, because MediaWiki messes with them...
 196         pagename = app['id'].replace('_', ' ')
 197
 198         # Drop a trailing newline, because mediawiki is going to drop it anyway
 199         # and it we don't we'll think the page has changed when it hasn't...
 200         if wikidata.endswith('\n'):
 201             wikidata = wikidata[:-1]
 202
 203         generated_pages[pagename] = wikidata
 204
 205         # Make a redirect from the name to the ID too, unless there's
 206         # already an existing page with the name and it isn't a redirect.
 207         noclobber = False
 208         apppagename = app['Name'].replace('_', ' ')
 209         apppagename = apppagename.replace('{', '')
 210         apppagename = apppagename.replace('}', ' ')
 211         apppagename = apppagename.replace(':', ' ')
 212         # Drop double spaces caused mostly by replacing ':' above
 213         apppagename = apppagename.replace('  ', ' ')
 214         for expagename in site.allpages(prefix=apppagename,
 215                 filterredir='nonredirects', generator=False):
 216             if expagename == apppagename:
 217                 noclobber = True
 218         # Another reason not to make the redirect page is if the app name
 219         # is the same as it's ID, because that will overwrite the real page
 220         # with an redirect to itself! (Although it seems like an odd
 221         # scenario this happens a lot, e.g. where there is metadata but no
 222         # builds or binaries to extract a name from.
 223         if apppagename == pagename:
 224             noclobber = True
 225         if not noclobber:
 226             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 227
 228     for tcat, genp in [(wikicat, generated_pages),
 229             (wikiredircat, generated_redirects)]:
 230         catpages = site.Pages['Category:' + tcat]
 231         existingpages = []
 232         for page in catpages:
 233             existingpages.append(page.name)
 234             if page.name in genp:
 235                 pagetxt = page.edit()
 236                 if pagetxt != genp[page.name]:
 237                     logging.info("Updating modified page " + page.name)
 238                     page.save(genp[page.name], summary='Auto-updated')
 239                 else:
 240                     logging.info("Page " + page.name + " is unchanged")
 241             else:
 242                 logging.info("Deleting page " + page.name)
 243                 page.delete('No longer published')
 244         for pagename, text in genp.items():
 245             logging.info("Checking " + pagename)
 246             if not pagename in existingpages:
 247                 logging.info("Creating page " + pagename)
 248                 try:
 249                     newpage = site.Pages[pagename]
 250                     newpage.save(text, summary='Auto-created')
 251                 except:
 252                     logging.warn("...FAILED to create page")
 253
 254     # Purge server cache to ensure counts are up to date
 255     site.pages['Repository Maintenance'].purge()
 256
 257 def delete_disabled_builds(apps, apkcache, repodirs):
 258     """Delete disabled build outputs.
 259
 260     :param apps: list of all applications, as per metadata.read_metadata
 261     :param apkcache: current apk cache information
 262     :param repodirs: the repo directories to process
 263     """
 264     for app in apps:
 265         for build in app['builds']:
 266             if 'disable' in build:
 267                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 268                 for repodir in repodirs:
 269                     apkpath = os.path.join(repodir, apkfilename)
 270                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 271                     for name in [apkpath, srcpath]:
 272                         if os.path.exists(name):
 273                             logging.info("Deleting disabled build output " + apkfilename)
 274                             os.remove(name)
 275                 if apkfilename in apkcache:
 276                     del apkcache[apkfilename]
 277
 278 def resize_icon(iconpath, density):
 279
 280     if not os.path.isfile(iconpath):
 281         return
 282
 283     try:
 284         im = Image.open(iconpath)
 285         size = dpi_to_px(density)
 286
 287         if any(length > size for length in im.size):
 288             oldsize = im.size
 289             im.thumbnail((size, size), Image.ANTIALIAS)
 290             logging.info(iconpath, "was too large at", oldsize, "- new size is", im.size)
 291             im.save(iconpath, "PNG")
 292
 293         else:
 294             logging.info(iconpath, "is small enough:", im.size)
 295
 296     except Exception,e:
 297         logging.info("WARNING: Failed resizing {0} - {1}".format(iconpath, e))
 298
 299 def resize_all_icons(repodirs):
 300     """Resize all icons that exceed the max size
 301
 302     :param repodirs: the repo directories to process
 303     """
 304     for repodir in repodirs:
 305         for density in get_densities():
 306             icon_dir = get_icon_dir(repodir, density)
 307             icon_glob = os.path.join(icon_dir, '*.png')
 308             for iconpath in glob.glob(icon_glob):
 309                 resize_icon(iconpath, density)
 310
 311 def scan_apks(apps, apkcache, repodir, knownapks):
 312     """Scan the apks in the given repo directory.
 313
 314     This also extracts the icons.
 315
 316     :param apps: list of all applications, as per metadata.read_metadata
 317     :param apkcache: current apk cache information
 318     :param repodir: repo directory to scan
 319     :param knownapks: known apks info
 320     :returns: (apks, cachechanged) where apks is a list of apk information,
 321               and cachechanged is True if the apkcache got changed.
 322     """
 323
 324     cachechanged = False
 325
 326     icon_dirs = get_icon_dirs(repodir)
 327     for icon_dir in icon_dirs:
 328         if os.path.exists(icon_dir):
 329             if options.clean:
 330                 shutil.rmtree(icon_dir)
 331                 os.makedirs(icon_dir)
 332         else:
 333             os.makedirs(icon_dir)
 334
 335     apks = []
 336     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 337     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 338     vername_pat = re.compile(".*versionName='([^']*)'.*")
 339     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 340     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 341     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 342     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 343     string_pat = re.compile(".*'([^']*)'.*")
 344     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 345
 346         apkfilename = apkfile[len(repodir) + 1:]
 347         if ' ' in apkfilename:
 348             logging.info("No spaces in APK filenames!")
 349             sys.exit(1)
 350
 351         if apkfilename in apkcache:
 352             logging.info("Reading " + apkfilename + " from cache")
 353             thisinfo = apkcache[apkfilename]
 354
 355         else:
 356
 357             if not options.quiet:
 358                 logging.info("Processing " + apkfilename)
 359             thisinfo = {}
 360             thisinfo['apkname'] = apkfilename
 361             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 362             if os.path.exists(os.path.join(repodir, srcfilename)):
 363                 thisinfo['srcname'] = srcfilename
 364             thisinfo['size'] = os.path.getsize(apkfile)
 365             thisinfo['permissions'] = []
 366             thisinfo['features'] = []
 367             thisinfo['icons_src'] = {}
 368             thisinfo['icons'] = {}
 369             p = FDroidPopen([os.path.join(config['sdk_path'],
 370                         'build-tools', config['build_tools'], 'aapt'),
 371                     'dump', 'badging', apkfile])
 372             if p.returncode != 0:
 373                 logging.critical("Failed to get apk information")
 374                 sys.exit(1)
 375             for line in p.stdout.splitlines():
 376                 if line.startswith("package:"):
 377                     try:
 378                         thisinfo['id'] = re.match(name_pat, line).group(1)
 379                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 380                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 381                     except Exception, e:
 382                         logging.info("Package matching failed: " + str(e))
 383                         logging.info("Line was: " + line)
 384                         sys.exit(1)
 385                 elif line.startswith("application:"):
 386                     thisinfo['name'] = re.match(label_pat, line).group(1)
 387                     # Keep path to non-dpi icon in case we need it
 388                     match = re.match(icon_pat_nodpi, line)
 389                     if match:
 390                         thisinfo['icons_src']['-1'] = match.group(1)
 391                 elif line.startswith("launchable-activity:"):
 392                     # Only use launchable-activity as fallback to application
 393                     if '-1' not in thisinfo['icons_src']:
 394                         match = re.match(icon_pat_nodpi, line)
 395                         if match:
 396                             thisinfo['icons_src']['-1'] = match.group(1)
 397                 elif line.startswith("application-icon-"):
 398                     match = re.match(icon_pat, line)
 399                     if match:
 400                         density = match.group(1)
 401                         path = match.group(2)
 402                         thisinfo['icons_src'][density] = path
 403                 elif line.startswith("sdkVersion:"):
 404                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 405                 elif line.startswith("native-code:"):
 406                     thisinfo['nativecode'] = []
 407                     for arch in line[13:].split(' '):
 408                         thisinfo['nativecode'].append(arch[1:-1])
 409                 elif line.startswith("uses-permission:"):
 410                     perm = re.match(string_pat, line).group(1)
 411                     if perm.startswith("android.permission."):
 412                         perm = perm[19:]
 413                     thisinfo['permissions'].append(perm)
 414                 elif line.startswith("uses-feature:"):
 415                     perm = re.match(string_pat, line).group(1)
 416                     #Filter out this, it's only added with the latest SDK tools and
 417                     #causes problems for lots of apps.
 418                     if (perm != "android.hardware.screen.portrait" and
 419                         perm != "android.hardware.screen.landscape"):
 420                         if perm.startswith("android.feature."):
 421                             perm = perm[16:]
 422                         thisinfo['features'].append(perm)
 423
 424             if not 'sdkversion' in thisinfo:
 425                 logging.info("  WARNING: no SDK version information found")
 426                 thisinfo['sdkversion'] = 0
 427
 428             # Check for debuggable apks...
 429             if common.isApkDebuggable(apkfile, config):
 430                 logging.info("WARNING: {0} is debuggable... {1}".format(apkfile, line))
 431
 432             # Calculate the sha256...
 433             sha = hashlib.sha256()
 434             with open(apkfile, 'rb') as f:
 435                 while True:
 436                     t = f.read(1024)
 437                     if len(t) == 0:
 438                         break
 439                     sha.update(t)
 440                 thisinfo['sha256'] = sha.hexdigest()
 441
 442             # Get the signature (or md5 of, to be precise)...
 443             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 444             if not os.path.exists(getsig_dir + "/getsig.class"):
 445                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 446                 sys.exit(1)
 447             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 448                         'getsig', os.path.join(os.getcwd(), apkfile)])
 449             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 450                 logging.critical("Failed to get apk signature")
 451                 sys.exit(1)
 452             thisinfo['sig'] = p.stdout[7:].strip()
 453
 454             apk = zipfile.ZipFile(apkfile, 'r')
 455
 456             iconfilename = "%s.%s.png" % (
 457                     thisinfo['id'],
 458                     thisinfo['versioncode'])
 459
 460             # Extract the icon file...
 461             densities = get_densities()
 462             empty_densities = []
 463             for density in densities:
 464                 if density not in thisinfo['icons_src']:
 465                     empty_densities.append(density)
 466                     continue
 467                 iconsrc = thisinfo['icons_src'][density]
 468                 icon_dir = get_icon_dir(repodir, density)
 469                 icondest = os.path.join(icon_dir, iconfilename)
 470
 471                 try:
 472                     iconfile = open(icondest, 'wb')
 473                     iconfile.write(apk.read(iconsrc))
 474                     iconfile.close()
 475                     thisinfo['icons'][density] = iconfilename
 476
 477                 except:
 478                     logging.info("WARNING: Error retrieving icon file")
 479                     del thisinfo['icons'][density]
 480                     del thisinfo['icons_src'][density]
 481                     empty_densities.append(density)
 482
 483             if '-1' in thisinfo['icons_src']:
 484                 iconsrc = thisinfo['icons_src']['-1']
 485                 iconpath = os.path.join(
 486                         get_icon_dir(repodir, None), iconfilename)
 487                 iconfile = open(iconpath, 'wb')
 488                 iconfile.write(apk.read(iconsrc))
 489                 iconfile.close()
 490                 try:
 491                     im = Image.open(iconpath)
 492                     dpi = px_to_dpi(im.size[0])
 493                     for density in densities:
 494                         if density in thisinfo['icons']:
 495                             break
 496                         if density == densities[-1] or dpi >= int(density):
 497                             thisinfo['icons'][density] = iconfilename
 498                             shutil.move(iconpath,
 499                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 500                             empty_densities.remove(density)
 501                             break
 502                 except Exception,e:
 503                     logging.info("WARNING: Failed reading {0} - {1}".format(iconpath, e))
 504
 505             if thisinfo['icons']:
 506                 thisinfo['icon'] = iconfilename
 507
 508             apk.close()
 509
 510             # First try resizing down to not lose quality
 511             last_density = None
 512             for density in densities:
 513                 if density not in empty_densities:
 514                     last_density = density
 515                     continue
 516                 if last_density is None:
 517                     continue
 518                 logging.info("Density %s not available, resizing down from %s" % (
 519                         density, last_density))
 520
 521                 last_iconpath = os.path.join(
 522                         get_icon_dir(repodir, last_density), iconfilename)
 523                 iconpath = os.path.join(
 524                         get_icon_dir(repodir, density), iconfilename)
 525                 try:
 526                     im = Image.open(last_iconpath)
 527                 except:
 528                     logging.info("WARNING: Invalid image file at %s" % last_iconpath)
 529                     continue
 530
 531                 size = dpi_to_px(density)
 532
 533                 im.thumbnail((size, size), Image.ANTIALIAS)
 534                 im.save(iconpath, "PNG")
 535                 empty_densities.remove(density)
 536
 537             # Then just copy from the highest resolution available
 538             last_density = None
 539             for density in reversed(densities):
 540                 if density not in empty_densities:
 541                     last_density = density
 542                     continue
 543                 if last_density is None:
 544                     continue
 545                 logging.info("Density %s not available, copying from lower density %s" % (
 546                         density, last_density))
 547
 548                 shutil.copyfile(
 549                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 550                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 551
 552                 empty_densities.remove(density)
 553
 554             for density in densities:
 555                 icon_dir = get_icon_dir(repodir, density)
 556                 icondest = os.path.join(icon_dir, iconfilename)
 557                 resize_icon(icondest, density)
 558
 559             # Copy from icons-mdpi to icons since mdpi is the baseline density
 560             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 561             if os.path.isfile(baseline):
 562                 shutil.copyfile(baseline,
 563                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 564
 565             # Record in known apks, getting the added date at the same time..
 566             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 567             if added:
 568                 thisinfo['added'] = added
 569
 570             apkcache[apkfilename] = thisinfo
 571             cachechanged = True
 572
 573         apks.append(thisinfo)
 574
 575     return apks, cachechanged
 576
 577
 578 repo_pubkey_fingerprint = None
 579
 580 def make_index(apps, apks, repodir, archive, categories):
 581     """Make a repo index.
 582
 583     :param apps: fully populated apps list
 584     :param apks: full populated apks list
 585     :param repodir: the repo directory
 586     :param archive: True if this is the archive repo, False if it's the
 587                     main one.
 588     :param categories: list of categories
 589     """
 590
 591     doc = Document()
 592
 593     def addElement(name, value, doc, parent):
 594         el = doc.createElement(name)
 595         el.appendChild(doc.createTextNode(value))
 596         parent.appendChild(el)
 597     def addElementCDATA(name, value, doc, parent):
 598         el = doc.createElement(name)
 599         el.appendChild(doc.createCDATASection(value))
 600         parent.appendChild(el)
 601
 602     root = doc.createElement("fdroid")
 603     doc.appendChild(root)
 604
 605     repoel = doc.createElement("repo")
 606
 607     if archive:
 608         repoel.setAttribute("name", config['archive_name'])
 609         if config['repo_maxage'] != 0:
 610             repoel.setAttribute("maxage", str(config['repo_maxage']))
 611         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 612         repoel.setAttribute("url", config['archive_url'])
 613         addElement('description', config['archive_description'], doc, repoel)
 614
 615     else:
 616         repoel.setAttribute("name", config['repo_name'])
 617         if config['repo_maxage'] != 0:
 618             repoel.setAttribute("maxage", str(config['repo_maxage']))
 619         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 620         repoel.setAttribute("url", config['repo_url'])
 621         addElement('description', config['repo_description'], doc, repoel)
 622
 623     repoel.setAttribute("version", "11")
 624     repoel.setAttribute("timestamp", str(int(time.time())))
 625
 626     if config['repo_keyalias']:
 627
 628         # Generate a certificate fingerprint the same way keytool does it
 629         # (but with slightly different formatting)
 630         def cert_fingerprint(data):
 631             digest = hashlib.sha1(data).digest()
 632             ret = []
 633             for i in range(4):
 634                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 635             return " ".join(ret)
 636
 637         def extract_pubkey():
 638             p = FDroidPopen(['keytool', '-exportcert',
 639                                   '-alias', config['repo_keyalias'],
 640                                   '-keystore', config['keystore'],
 641                                   '-storepass', config['keystorepass']])
 642             if p.returncode != 0:
 643                 logging.critical("Failed to get repo pubkey")
 644                 sys.exit(1)
 645             global repo_pubkey_fingerprint
 646             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 647             return "".join("%02x" % ord(b) for b in p.stdout)
 648
 649         repoel.setAttribute("pubkey", extract_pubkey())
 650
 651     root.appendChild(repoel)
 652
 653     for app in apps:
 654
 655         if app['Disabled'] is not None:
 656             continue
 657
 658         # Get a list of the apks for this app...
 659         apklist = []
 660         for apk in apks:
 661             if apk['id'] == app['id']:
 662                 apklist.append(apk)
 663
 664         if len(apklist) == 0:
 665             continue
 666
 667         apel = doc.createElement("application")
 668         apel.setAttribute("id", app['id'])
 669         root.appendChild(apel)
 670
 671         addElement('id', app['id'], doc, apel)
 672         if 'added' in app:
 673             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 674         if 'lastupdated' in app:
 675             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 676         addElement('name', app['Name'], doc, apel)
 677         addElement('summary', app['Summary'], doc, apel)
 678         if app['icon']:
 679             addElement('icon', app['icon'], doc, apel)
 680         def linkres(link):
 681             for app in apps:
 682                 if app['id'] == link:
 683                     return ("fdroid.app:" + link, app['Name'])
 684             raise MetaDataException("Cannot resolve app id " + link)
 685         addElement('desc',
 686                 metadata.description_html(app['Description'], linkres), doc, apel)
 687         addElement('license', app['License'], doc, apel)
 688         if 'Categories' in app:
 689             appcategories = [c.strip() for c in app['Categories'].split(',')]
 690             addElement('categories', ','.join(appcategories), doc, apel)
 691             # We put the first (primary) category in LAST, which will have
 692             # the desired effect of making clients that only understand one
 693             # category see that one.
 694             addElement('category', appcategories[0], doc, apel)
 695         addElement('web', app['Web Site'], doc, apel)
 696         addElement('source', app['Source Code'], doc, apel)
 697         addElement('tracker', app['Issue Tracker'], doc, apel)
 698         if app['Donate']:
 699             addElement('donate', app['Donate'], doc, apel)
 700         if app['Bitcoin']:
 701             addElement('bitcoin', app['Bitcoin'], doc, apel)
 702         if app['Litecoin']:
 703             addElement('litecoin', app['Litecoin'], doc, apel)
 704         if app['Dogecoin']:
 705             addElement('dogecoin', app['Dogecoin'], doc, apel)
 706         if app['FlattrID']:
 707             addElement('flattr', app['FlattrID'], doc, apel)
 708
 709         # These elements actually refer to the current version (i.e. which
 710         # one is recommended. They are historically mis-named, and need
 711         # changing, but stay like this for now to support existing clients.
 712         addElement('marketversion', app['Current Version'], doc, apel)
 713         addElement('marketvercode', app['Current Version Code'], doc, apel)
 714
 715         if app['AntiFeatures']:
 716             af = app['AntiFeatures'].split(',')
 717             # TODO: Temporarily not including UpstreamNonFree in the index,
 718             # because current F-Droid clients do not understand it, and also
 719             # look ugly when they encounter an unknown antifeature. This
 720             # filtering can be removed in time...
 721             if 'UpstreamNonFree' in af:
 722                 af.remove('UpstreamNonFree')
 723             if af:
 724                 addElement('antifeatures', ','.join(af), doc, apel)
 725         if app['Provides']:
 726             pv = app['Provides'].split(',')
 727             addElement('provides', ','.join(pv), doc, apel)
 728         if app['Requires Root']:
 729             addElement('requirements', 'root', doc, apel)
 730
 731         # Sort the apk list into version order, just so the web site
 732         # doesn't have to do any work by default...
 733         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 734
 735         # Check for duplicates - they will make the client unhappy...
 736         for i in range(len(apklist) - 1):
 737             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 738                 logging.critical("duplicate versions: '%s' - '%s'" % (
 739                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 740                 sys.exit(1)
 741
 742         for apk in apklist:
 743             apkel = doc.createElement("package")
 744             apel.appendChild(apkel)
 745             addElement('version', apk['version'], doc, apkel)
 746             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 747             addElement('apkname', apk['apkname'], doc, apkel)
 748             if 'srcname' in apk:
 749                 addElement('srcname', apk['srcname'], doc, apkel)
 750             for hash_type in ['sha256']:
 751                 if not hash_type in apk:
 752                     continue
 753                 hashel = doc.createElement("hash")
 754                 hashel.setAttribute("type", hash_type)
 755                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 756                 apkel.appendChild(hashel)
 757             addElement('sig', apk['sig'], doc, apkel)
 758             addElement('size', str(apk['size']), doc, apkel)
 759             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 760             if 'added' in apk:
 761                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 762             if app['Requires Root']:
 763                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 764                     apk['permissions'].append('ACCESS_SUPERUSER')
 765
 766             if len(apk['permissions']) > 0:
 767                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 768             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 769                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 770             if len(apk['features']) > 0:
 771                 addElement('features', ','.join(apk['features']), doc, apkel)
 772
 773     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 774     if options.pretty:
 775         output = doc.toprettyxml()
 776     else:
 777         output = doc.toxml()
 778     of.write(output)
 779     of.close()
 780
 781     if config['repo_keyalias'] is not None:
 782
 783         if not options.quiet:
 784             logging.info("Creating signed index.")
 785             logging.info("Key fingerprint:", repo_pubkey_fingerprint)
 786
 787         #Create a jar of the index...
 788         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 789         if p.returncode != 0:
 790             logging.critical("Failed to create jar file")
 791             sys.exit(1)
 792
 793         # Sign the index...
 794         p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
 795             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 796             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 797             os.path.join(repodir, 'index.jar') , config['repo_keyalias']])
 798         if p.returncode != 0:
 799             logging.info("Failed to sign index")
 800             sys.exit(1)
 801
 802     # Copy the repo icon into the repo directory...
 803     icon_dir = os.path.join(repodir ,'icons')
 804     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 805     shutil.copyfile(config['repo_icon'], iconfilename)
 806
 807     # Write a category list in the repo to allow quick access...
 808     catdata = ''
 809     for cat in categories:
 810         catdata += cat + '\n'
 811     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 812     f.write(catdata)
 813     f.close()
 814
 815
 816
 817 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 818
 819     for app in apps:
 820
 821         # Get a list of the apks for this app...
 822         apklist = []
 823         for apk in apks:
 824             if apk['id'] == app['id']:
 825                 apklist.append(apk)
 826
 827         # Sort the apk list into version order...
 828         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 829
 830         if app['Archive Policy']:
 831             keepversions = int(app['Archive Policy'][:-9])
 832         else:
 833             keepversions = defaultkeepversions
 834
 835         if len(apklist) > keepversions:
 836             for apk in apklist[keepversions:]:
 837                 logging.info("Moving " + apk['apkname'] + " to archive")
 838                 shutil.move(os.path.join(repodir, apk['apkname']),
 839                     os.path.join(archivedir, apk['apkname']))
 840                 if 'srcname' in apk:
 841                     shutil.move(os.path.join(repodir, apk['srcname']),
 842                         os.path.join(archivedir, apk['srcname']))
 843                 archapks.append(apk)
 844                 apks.remove(apk)
 845
 846
 847 config = None
 848 options = None
 849
 850 def main():
 851
 852     global config, options
 853
 854     # Parse command line...
 855     parser = OptionParser()
 856     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 857                       help="Create skeleton metadata files that are missing")
 858     parser.add_option("-v", "--verbose", action="store_true", default=False,
 859                       help="Spew out even more information than normal")
 860     parser.add_option("-q", "--quiet", action="store_true", default=False,
 861                       help="No output, except for warnings and errors")
 862     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 863                       help="Report on build data status")
 864     parser.add_option("-i", "--interactive", default=False, action="store_true",
 865                       help="Interactively ask about things that need updating.")
 866     parser.add_option("-I", "--icons", action="store_true", default=False,
 867                       help="Resize all the icons exceeding the max pixel size and exit")
 868     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 869                       help="Specify editor to use in interactive mode. Default "+
 870                           "is /etc/alternatives/editor")
 871     parser.add_option("-w", "--wiki", default=False, action="store_true",
 872                       help="Update the wiki")
 873     parser.add_option("", "--pretty", action="store_true", default=False,
 874                       help="Produce human-readable index.xml")
 875     parser.add_option("--clean", action="store_true", default=False,
 876                       help="Clean update - don't uses caches, reprocess all apks")
 877     (options, args) = parser.parse_args()
 878
 879     config = common.read_config(options)
 880
 881     repodirs = ['repo']
 882     if config['archive_older'] != 0:
 883         repodirs.append('archive')
 884         if not os.path.exists('archive'):
 885             os.mkdir('archive')
 886
 887     if options.icons:
 888         resize_all_icons(repodirs)
 889         sys.exit(0)
 890
 891     # Get all apps...
 892     apps = metadata.read_metadata()
 893
 894     # Generate a list of categories...
 895     categories = []
 896     for app in apps:
 897         cats = app['Categories'].split(',')
 898         for cat in cats:
 899             if cat not in categories:
 900                 categories.append(cat)
 901
 902     # Read known apks data (will be updated and written back when we've finished)
 903     knownapks = common.KnownApks()
 904
 905     # Gather information about all the apk files in the repo directory, using
 906     # cached data if possible.
 907     apkcachefile = os.path.join('tmp', 'apkcache')
 908     if not options.clean and os.path.exists(apkcachefile):
 909         with open(apkcachefile, 'rb') as cf:
 910             apkcache = pickle.load(cf)
 911     else:
 912         apkcache = {}
 913     cachechanged = False
 914
 915     delete_disabled_builds(apps, apkcache, repodirs)
 916
 917     # Scan all apks in the main repo
 918     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 919     if cc:
 920         cachechanged = True
 921
 922     # Scan the archive repo for apks as well
 923     if len(repodirs) > 1:
 924         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 925         if cc:
 926             cachechanged = True
 927     else:
 928         archapks = []
 929
 930     # Some information from the apks needs to be applied up to the application
 931     # level. When doing this, we use the info from the most recent version's apk.
 932     # We deal with figuring out when the app was added and last updated at the
 933     # same time.
 934     for app in apps:
 935         bestver = 0
 936         added = None
 937         lastupdated = None
 938         for apk in apks + archapks:
 939             if apk['id'] == app['id']:
 940                 if apk['versioncode'] > bestver:
 941                     bestver = apk['versioncode']
 942                     bestapk = apk
 943
 944                 if 'added' in apk:
 945                     if not added or apk['added'] < added:
 946                         added = apk['added']
 947                     if not lastupdated or apk['added'] > lastupdated:
 948                         lastupdated = apk['added']
 949
 950         if added:
 951             app['added'] = added
 952         else:
 953             logging.info("WARNING: Don't know when " + app['id'] + " was added")
 954         if lastupdated:
 955             app['lastupdated'] = lastupdated
 956         else:
 957             logging.info("WARNING: Don't know when " + app['id'] + " was last updated")
 958
 959         if bestver == 0:
 960             if app['Name'] is None:
 961                 app['Name'] = app['id']
 962             app['icon'] = None
 963             logging.info("WARNING: Application " + app['id'] + " has no packages")
 964         else:
 965             if app['Name'] is None:
 966                 app['Name'] = bestapk['name']
 967             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 968
 969     # Sort the app list by name, then the web site doesn't have to by default.
 970     # (we had to wait until we'd scanned the apks to do this, because mostly the
 971     # name comes from there!)
 972     apps = sorted(apps, key=lambda app: app['Name'].upper())
 973
 974     # Generate warnings for apk's with no metadata (or create skeleton
 975     # metadata files, if requested on the command line)
 976     for apk in apks:
 977         found = False
 978         for app in apps:
 979             if app['id'] == apk['id']:
 980                 found = True
 981                 break
 982         if not found:
 983             if options.createmeta:
 984                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 985                 f.write("License:Unknown\n")
 986                 f.write("Web Site:\n")
 987                 f.write("Source Code:\n")
 988                 f.write("Issue Tracker:\n")
 989                 f.write("Summary:" + apk['name'] + "\n")
 990                 f.write("Description:\n")
 991                 f.write(apk['name'] + "\n")
 992                 f.write(".\n")
 993                 f.close()
 994                 logging.info("Generated skeleton metadata for " + apk['id'])
 995             else:
 996                 logging.info("WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata")
 997                 logging.info("       " + apk['name'] + " - " + apk['version'])
 998
 999     if len(repodirs) > 1:
1000         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1001
1002     # Make the index for the main repo...
1003     make_index(apps, apks, repodirs[0], False, categories)
1004
1005     # If there's an archive repo,  make the index for it. We already scanned it
1006     # earlier on.
1007     if len(repodirs) > 1:
1008         make_index(apps, archapks, repodirs[1], True, categories)
1009
1010     if config['update_stats']:
1011
1012         # Update known apks info...
1013         knownapks.writeifchanged()
1014
1015         # Generate latest apps data for widget
1016         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1017             data = ''
1018             for line in file(os.path.join('stats', 'latestapps.txt')):
1019                 appid = line.rstrip()
1020                 data += appid + "\t"
1021                 for app in apps:
1022                     if app['id'] == appid:
1023                         data += app['Name'] + "\t"
1024                         if app['icon'] is not None:
1025                             data += app['icon'] + "\t"
1026                         data += app['License'] + "\n"
1027                         break
1028             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1029             f.write(data)
1030             f.close()
1031
1032     if cachechanged:
1033         with open(apkcachefile, 'wb') as cf:
1034             pickle.dump(apkcache, cf)
1035
1036     # Update the wiki...
1037     if options.wiki:
1038         update_wiki(apps, apks + archapks)
1039
1040     logging.info("Finished.")
1041
1042 if __name__ == "__main__":
1043     main()
1044