fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         if app['Current Version Code'] == '0':
 145             cantupdate = True
 146         # Sort with most recent first...
 147         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 148
 149         wikidata += "=Versions=\n"
 150         if len(apklist) == 0:
 151             wikidata += "We currently have no versions of this app available."
 152         elif not gotcurrentver:
 153             wikidata += "We don't have the current version of this app."
 154         else:
 155             wikidata += "We have the current version of this app."
 156         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 157         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 158         if len(app['No Source Since']) > 0:
 159             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 160         if len(app['Current Version']) > 0:
 161             wikidata += "The current (recommended) version is " + app['Current Version']
 162             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 163         validapks = 0
 164         for apk in apklist:
 165             wikidata += "==" + apk['version'] + "==\n"
 166
 167             if 'buildproblem' in apk:
 168                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 169             else:
 170                 validapks += 1
 171                 wikidata += "This version is built and signed by "
 172                 if 'srcname' in apk:
 173                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 174                 else:
 175                     wikidata += "the original developer.\n\n"
 176             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 177
 178         wikidata += '\n[[Category:' + wikicat + ']]\n'
 179         if len(app['No Source Since']) > 0:
 180             wikidata += '\n[[Category:Apps missing source code]]\n'
 181         if validapks == 0 and not app['Disabled']:
 182             wikidata += '\n[[Category:Apps with no packages]]\n'
 183         if cantupdate and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps we can't update]]\n"
 185         if buildfails and not app['Disabled']:
 186             wikidata += "\n[[Category:Apps with failing builds]]\n"
 187         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 188             wikidata += '\n[[Category:Apps to Update]]\n'
 189         if app['Disabled']:
 190             wikidata += '\n[[Category:Apps that are disabled]]\n'
 191         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 192             wikidata += '\n[[Category:Apps with no update check]]\n'
 193         for appcat in app['Categories']:
 194             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 195
 196         # We can't have underscores in the page name, even if they're in
 197         # the package ID, because MediaWiki messes with them...
 198         pagename = app['id'].replace('_', ' ')
 199
 200         # Drop a trailing newline, because mediawiki is going to drop it anyway
 201         # and it we don't we'll think the page has changed when it hasn't...
 202         if wikidata.endswith('\n'):
 203             wikidata = wikidata[:-1]
 204
 205         generated_pages[pagename] = wikidata
 206
 207         # Make a redirect from the name to the ID too, unless there's
 208         # already an existing page with the name and it isn't a redirect.
 209         noclobber = False
 210         apppagename = app['Name'].replace('_', ' ')
 211         apppagename = apppagename.replace('{', '')
 212         apppagename = apppagename.replace('}', ' ')
 213         apppagename = apppagename.replace(':', ' ')
 214         # Drop double spaces caused mostly by replacing ':' above
 215         apppagename = apppagename.replace('  ', ' ')
 216         for expagename in site.allpages(prefix=apppagename,
 217                 filterredir='nonredirects', generator=False):
 218             if expagename == apppagename:
 219                 noclobber = True
 220         # Another reason not to make the redirect page is if the app name
 221         # is the same as it's ID, because that will overwrite the real page
 222         # with an redirect to itself! (Although it seems like an odd
 223         # scenario this happens a lot, e.g. where there is metadata but no
 224         # builds or binaries to extract a name from.
 225         if apppagename == pagename:
 226             noclobber = True
 227         if not noclobber:
 228             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 229
 230     for tcat, genp in [(wikicat, generated_pages),
 231             (wikiredircat, generated_redirects)]:
 232         catpages = site.Pages['Category:' + tcat]
 233         existingpages = []
 234         for page in catpages:
 235             existingpages.append(page.name)
 236             if page.name in genp:
 237                 pagetxt = page.edit()
 238                 if pagetxt != genp[page.name]:
 239                     logging.debug("Updating modified page " + page.name)
 240                     page.save(genp[page.name], summary='Auto-updated')
 241                 else:
 242                     logging.debug("Page " + page.name + " is unchanged")
 243             else:
 244                 logging.warn("Deleting page " + page.name)
 245                 page.delete('No longer published')
 246         for pagename, text in genp.items():
 247             logging.debug("Checking " + pagename)
 248             if not pagename in existingpages:
 249                 logging.debug("Creating page " + pagename)
 250                 try:
 251                     newpage = site.Pages[pagename]
 252                     newpage.save(text, summary='Auto-created')
 253                 except:
 254                     logging.error("...FAILED to create page")
 255
 256     # Purge server cache to ensure counts are up to date
 257     site.pages['Repository Maintenance'].purge()
 258
 259 def delete_disabled_builds(apps, apkcache, repodirs):
 260     """Delete disabled build outputs.
 261
 262     :param apps: list of all applications, as per metadata.read_metadata
 263     :param apkcache: current apk cache information
 264     :param repodirs: the repo directories to process
 265     """
 266     for app in apps:
 267         for build in app['builds']:
 268             if 'disable' in build:
 269                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 270                 for repodir in repodirs:
 271                     apkpath = os.path.join(repodir, apkfilename)
 272                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 273                     for name in [apkpath, srcpath]:
 274                         if os.path.exists(name):
 275                             logging.warn("Deleting disabled build output " + apkfilename)
 276                             os.remove(name)
 277                 if apkfilename in apkcache:
 278                     del apkcache[apkfilename]
 279
 280 def resize_icon(iconpath, density):
 281
 282     if not os.path.isfile(iconpath):
 283         return
 284
 285     try:
 286         im = Image.open(iconpath)
 287         size = dpi_to_px(density)
 288
 289         if any(length > size for length in im.size):
 290             oldsize = im.size
 291             im.thumbnail((size, size), Image.ANTIALIAS)
 292             logging.debug("%s was too large at %s - new size is %s" % (
 293                 iconpath, oldsize, im.size))
 294             im.save(iconpath, "PNG")
 295
 296         else:
 297             logging.info("%s is small enough: %s" % im.size)
 298
 299     except Exception,e:
 300         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 301
 302 def resize_all_icons(repodirs):
 303     """Resize all icons that exceed the max size
 304
 305     :param repodirs: the repo directories to process
 306     """
 307     for repodir in repodirs:
 308         for density in get_densities():
 309             icon_dir = get_icon_dir(repodir, density)
 310             icon_glob = os.path.join(icon_dir, '*.png')
 311             for iconpath in glob.glob(icon_glob):
 312                 resize_icon(iconpath, density)
 313
 314 def scan_apks(apps, apkcache, repodir, knownapks):
 315     """Scan the apks in the given repo directory.
 316
 317     This also extracts the icons.
 318
 319     :param apps: list of all applications, as per metadata.read_metadata
 320     :param apkcache: current apk cache information
 321     :param repodir: repo directory to scan
 322     :param knownapks: known apks info
 323     :returns: (apks, cachechanged) where apks is a list of apk information,
 324               and cachechanged is True if the apkcache got changed.
 325     """
 326
 327     cachechanged = False
 328
 329     icon_dirs = get_icon_dirs(repodir)
 330     for icon_dir in icon_dirs:
 331         if os.path.exists(icon_dir):
 332             if options.clean:
 333                 shutil.rmtree(icon_dir)
 334                 os.makedirs(icon_dir)
 335         else:
 336             os.makedirs(icon_dir)
 337
 338     apks = []
 339     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 340     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 341     vername_pat = re.compile(".*versionName='([^']*)'.*")
 342     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 343     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 344     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 345     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 346     string_pat = re.compile(".*'([^']*)'.*")
 347     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 348
 349         apkfilename = apkfile[len(repodir) + 1:]
 350         if ' ' in apkfilename:
 351             logging.error("No spaces in APK filenames!")
 352             sys.exit(1)
 353
 354         if apkfilename in apkcache:
 355             logging.debug("Reading " + apkfilename + " from cache")
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             logging.info("Processing " + apkfilename)
 361             thisinfo = {}
 362             thisinfo['apkname'] = apkfilename
 363             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 364             if os.path.exists(os.path.join(repodir, srcfilename)):
 365                 thisinfo['srcname'] = srcfilename
 366             thisinfo['size'] = os.path.getsize(apkfile)
 367             thisinfo['permissions'] = []
 368             thisinfo['features'] = []
 369             thisinfo['icons_src'] = {}
 370             thisinfo['icons'] = {}
 371             p = FDroidPopen([os.path.join(config['sdk_path'],
 372                         'build-tools', config['build_tools'], 'aapt'),
 373                     'dump', 'badging', apkfile])
 374             if p.returncode != 0:
 375                 logging.critical("Failed to get apk information")
 376                 sys.exit(1)
 377             for line in p.stdout.splitlines():
 378                 if line.startswith("package:"):
 379                     try:
 380                         thisinfo['id'] = re.match(name_pat, line).group(1)
 381                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 382                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 383                     except Exception, e:
 384                         logging.info("Package matching failed: " + str(e))
 385                         logging.info("Line was: " + line)
 386                         sys.exit(1)
 387                 elif line.startswith("application:"):
 388                     thisinfo['name'] = re.match(label_pat, line).group(1)
 389                     # Keep path to non-dpi icon in case we need it
 390                     match = re.match(icon_pat_nodpi, line)
 391                     if match:
 392                         thisinfo['icons_src']['-1'] = match.group(1)
 393                 elif line.startswith("launchable-activity:"):
 394                     # Only use launchable-activity as fallback to application
 395                     if '-1' not in thisinfo['icons_src']:
 396                         match = re.match(icon_pat_nodpi, line)
 397                         if match:
 398                             thisinfo['icons_src']['-1'] = match.group(1)
 399                 elif line.startswith("application-icon-"):
 400                     match = re.match(icon_pat, line)
 401                     if match:
 402                         density = match.group(1)
 403                         path = match.group(2)
 404                         thisinfo['icons_src'][density] = path
 405                 elif line.startswith("sdkVersion:"):
 406                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 407                 elif line.startswith("maxSdkVersion:"):
 408                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 logging.warn("no SDK version information found")
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 450                 sys.exit(1)
 451             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 452                         'getsig', os.path.join(os.getcwd(), apkfile)])
 453             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 454                 logging.critical("Failed to get apk signature")
 455                 sys.exit(1)
 456             thisinfo['sig'] = p.stdout[7:].strip()
 457
 458             apk = zipfile.ZipFile(apkfile, 'r')
 459
 460             iconfilename = "%s.%s.png" % (
 461                     thisinfo['id'],
 462                     thisinfo['versioncode'])
 463
 464             # Extract the icon file...
 465             densities = get_densities()
 466             empty_densities = []
 467             for density in densities:
 468                 if density not in thisinfo['icons_src']:
 469                     empty_densities.append(density)
 470                     continue
 471                 iconsrc = thisinfo['icons_src'][density]
 472                 icon_dir = get_icon_dir(repodir, density)
 473                 icondest = os.path.join(icon_dir, iconfilename)
 474
 475                 try:
 476                     iconfile = open(icondest, 'wb')
 477                     iconfile.write(apk.read(iconsrc))
 478                     iconfile.close()
 479                     thisinfo['icons'][density] = iconfilename
 480
 481                 except:
 482                     logging.warn("Error retrieving icon file")
 483                     del thisinfo['icons'][density]
 484                     del thisinfo['icons_src'][density]
 485                     empty_densities.append(density)
 486
 487             if '-1' in thisinfo['icons_src']:
 488                 iconsrc = thisinfo['icons_src']['-1']
 489                 iconpath = os.path.join(
 490                         get_icon_dir(repodir, None), iconfilename)
 491                 iconfile = open(iconpath, 'wb')
 492                 iconfile.write(apk.read(iconsrc))
 493                 iconfile.close()
 494                 try:
 495                     im = Image.open(iconpath)
 496                     dpi = px_to_dpi(im.size[0])
 497                     for density in densities:
 498                         if density in thisinfo['icons']:
 499                             break
 500                         if density == densities[-1] or dpi >= int(density):
 501                             thisinfo['icons'][density] = iconfilename
 502                             shutil.move(iconpath,
 503                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 504                             empty_densities.remove(density)
 505                             break
 506                 except Exception,e:
 507                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 508
 509             if thisinfo['icons']:
 510                 thisinfo['icon'] = iconfilename
 511
 512             apk.close()
 513
 514             # First try resizing down to not lose quality
 515             last_density = None
 516             for density in densities:
 517                 if density not in empty_densities:
 518                     last_density = density
 519                     continue
 520                 if last_density is None:
 521                     continue
 522                 logging.info("Density %s not available, resizing down from %s" % (
 523                         density, last_density))
 524
 525                 last_iconpath = os.path.join(
 526                         get_icon_dir(repodir, last_density), iconfilename)
 527                 iconpath = os.path.join(
 528                         get_icon_dir(repodir, density), iconfilename)
 529                 try:
 530                     im = Image.open(last_iconpath)
 531                 except:
 532                     logging.warn("Invalid image file at %s" % last_iconpath)
 533                     continue
 534
 535                 size = dpi_to_px(density)
 536
 537                 im.thumbnail((size, size), Image.ANTIALIAS)
 538                 im.save(iconpath, "PNG")
 539                 empty_densities.remove(density)
 540
 541             # Then just copy from the highest resolution available
 542             last_density = None
 543             for density in reversed(densities):
 544                 if density not in empty_densities:
 545                     last_density = density
 546                     continue
 547                 if last_density is None:
 548                     continue
 549                 logging.info("Density %s not available, copying from lower density %s" % (
 550                         density, last_density))
 551
 552                 shutil.copyfile(
 553                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 554                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 555
 556                 empty_densities.remove(density)
 557
 558             for density in densities:
 559                 icon_dir = get_icon_dir(repodir, density)
 560                 icondest = os.path.join(icon_dir, iconfilename)
 561                 resize_icon(icondest, density)
 562
 563             # Copy from icons-mdpi to icons since mdpi is the baseline density
 564             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 565             if os.path.isfile(baseline):
 566                 shutil.copyfile(baseline,
 567                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 568
 569             # Record in known apks, getting the added date at the same time..
 570             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 571             if added:
 572                 thisinfo['added'] = added
 573
 574             apkcache[apkfilename] = thisinfo
 575             cachechanged = True
 576
 577         apks.append(thisinfo)
 578
 579     return apks, cachechanged
 580
 581
 582 repo_pubkey_fingerprint = None
 583
 584 def make_index(apps, apks, repodir, archive, categories):
 585     """Make a repo index.
 586
 587     :param apps: fully populated apps list
 588     :param apks: full populated apks list
 589     :param repodir: the repo directory
 590     :param archive: True if this is the archive repo, False if it's the
 591                     main one.
 592     :param categories: list of categories
 593     """
 594
 595     doc = Document()
 596
 597     def addElement(name, value, doc, parent):
 598         el = doc.createElement(name)
 599         el.appendChild(doc.createTextNode(value))
 600         parent.appendChild(el)
 601     def addElementCDATA(name, value, doc, parent):
 602         el = doc.createElement(name)
 603         el.appendChild(doc.createCDATASection(value))
 604         parent.appendChild(el)
 605
 606     root = doc.createElement("fdroid")
 607     doc.appendChild(root)
 608
 609     repoel = doc.createElement("repo")
 610
 611     if archive:
 612         repoel.setAttribute("name", config['archive_name'])
 613         if config['repo_maxage'] != 0:
 614             repoel.setAttribute("maxage", str(config['repo_maxage']))
 615         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 616         repoel.setAttribute("url", config['archive_url'])
 617         addElement('description', config['archive_description'], doc, repoel)
 618
 619     else:
 620         repoel.setAttribute("name", config['repo_name'])
 621         if config['repo_maxage'] != 0:
 622             repoel.setAttribute("maxage", str(config['repo_maxage']))
 623         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 624         repoel.setAttribute("url", config['repo_url'])
 625         addElement('description', config['repo_description'], doc, repoel)
 626
 627     repoel.setAttribute("version", "12")
 628     repoel.setAttribute("timestamp", str(int(time.time())))
 629
 630     if 'repo_keyalias' in config:
 631
 632         # Generate a certificate fingerprint the same way keytool does it
 633         # (but with slightly different formatting)
 634         def cert_fingerprint(data):
 635             digest = hashlib.sha1(data).digest()
 636             ret = []
 637             for i in range(4):
 638                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 639             return " ".join(ret)
 640
 641         def extract_pubkey():
 642             p = FDroidPopen(['keytool', '-exportcert',
 643                                   '-alias', config['repo_keyalias'],
 644                                   '-keystore', config['keystore'],
 645                                   '-storepass:file', config['keystorepassfile']])
 646             if p.returncode != 0:
 647                 logging.critical("Failed to get repo pubkey")
 648                 sys.exit(1)
 649             global repo_pubkey_fingerprint
 650             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 651             return "".join("%02x" % ord(b) for b in p.stdout)
 652
 653         repoel.setAttribute("pubkey", extract_pubkey())
 654
 655     root.appendChild(repoel)
 656
 657     for app in apps:
 658
 659         if app['Disabled'] is not None:
 660             continue
 661
 662         # Get a list of the apks for this app...
 663         apklist = []
 664         for apk in apks:
 665             if apk['id'] == app['id']:
 666                 apklist.append(apk)
 667
 668         if len(apklist) == 0:
 669             continue
 670
 671         apel = doc.createElement("application")
 672         apel.setAttribute("id", app['id'])
 673         root.appendChild(apel)
 674
 675         addElement('id', app['id'], doc, apel)
 676         if 'added' in app:
 677             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 678         if 'lastupdated' in app:
 679             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 680         addElement('name', app['Name'], doc, apel)
 681         addElement('summary', app['Summary'], doc, apel)
 682         if app['icon']:
 683             addElement('icon', app['icon'], doc, apel)
 684         def linkres(link):
 685             for app in apps:
 686                 if app['id'] == link:
 687                     return ("fdroid.app:" + link, app['Name'])
 688             raise MetaDataException("Cannot resolve app id " + link)
 689         addElement('desc',
 690                 metadata.description_html(app['Description'], linkres), doc, apel)
 691         addElement('license', app['License'], doc, apel)
 692         if 'Categories' in app:
 693             addElement('categories', ','.join(app["Categories"]), doc, apel)
 694             # We put the first (primary) category in LAST, which will have
 695             # the desired effect of making clients that only understand one
 696             # category see that one.
 697             addElement('category', app["Categories"][0], doc, apel)
 698         addElement('web', app['Web Site'], doc, apel)
 699         addElement('source', app['Source Code'], doc, apel)
 700         addElement('tracker', app['Issue Tracker'], doc, apel)
 701         if app['Donate']:
 702             addElement('donate', app['Donate'], doc, apel)
 703         if app['Bitcoin']:
 704             addElement('bitcoin', app['Bitcoin'], doc, apel)
 705         if app['Litecoin']:
 706             addElement('litecoin', app['Litecoin'], doc, apel)
 707         if app['Dogecoin']:
 708             addElement('dogecoin', app['Dogecoin'], doc, apel)
 709         if app['FlattrID']:
 710             addElement('flattr', app['FlattrID'], doc, apel)
 711
 712         # These elements actually refer to the current version (i.e. which
 713         # one is recommended. They are historically mis-named, and need
 714         # changing, but stay like this for now to support existing clients.
 715         addElement('marketversion', app['Current Version'], doc, apel)
 716         addElement('marketvercode', app['Current Version Code'], doc, apel)
 717
 718         if app['AntiFeatures']:
 719             af = app['AntiFeatures'].split(',')
 720             # TODO: Temporarily not including UpstreamNonFree in the index,
 721             # because current F-Droid clients do not understand it, and also
 722             # look ugly when they encounter an unknown antifeature. This
 723             # filtering can be removed in time...
 724             if 'UpstreamNonFree' in af:
 725                 af.remove('UpstreamNonFree')
 726             if af:
 727                 addElement('antifeatures', ','.join(af), doc, apel)
 728         if app['Provides']:
 729             pv = app['Provides'].split(',')
 730             addElement('provides', ','.join(pv), doc, apel)
 731         if app['Requires Root']:
 732             addElement('requirements', 'root', doc, apel)
 733
 734         # Sort the apk list into version order, just so the web site
 735         # doesn't have to do any work by default...
 736         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 737
 738         # Check for duplicates - they will make the client unhappy...
 739         for i in range(len(apklist) - 1):
 740             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 741                 logging.critical("duplicate versions: '%s' - '%s'" % (
 742                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 743                 sys.exit(1)
 744
 745         for apk in apklist:
 746             apkel = doc.createElement("package")
 747             apel.appendChild(apkel)
 748             addElement('version', apk['version'], doc, apkel)
 749             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 750             addElement('apkname', apk['apkname'], doc, apkel)
 751             if 'srcname' in apk:
 752                 addElement('srcname', apk['srcname'], doc, apkel)
 753             for hash_type in ['sha256']:
 754                 if not hash_type in apk:
 755                     continue
 756                 hashel = doc.createElement("hash")
 757                 hashel.setAttribute("type", hash_type)
 758                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 759                 apkel.appendChild(hashel)
 760             addElement('sig', apk['sig'], doc, apkel)
 761             addElement('size', str(apk['size']), doc, apkel)
 762             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 763             if 'maxsdkversion' in apk:
 764                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 765             if 'added' in apk:
 766                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 767             if app['Requires Root']:
 768                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 769                     apk['permissions'].append('ACCESS_SUPERUSER')
 770
 771             if len(apk['permissions']) > 0:
 772                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 773             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 774                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 775             if len(apk['features']) > 0:
 776                 addElement('features', ','.join(apk['features']), doc, apkel)
 777
 778     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 779     if options.pretty:
 780         output = doc.toprettyxml()
 781     else:
 782         output = doc.toxml()
 783     of.write(output)
 784     of.close()
 785
 786     if 'repo_keyalias' in config:
 787
 788         logging.info("Creating signed index.")
 789         logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint)
 790
 791         #Create a jar of the index...
 792         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 793         if p.returncode != 0:
 794             logging.critical("Failed to create jar file")
 795             sys.exit(1)
 796
 797         # Sign the index...
 798         p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
 799             '-storepass:file', config['keystorepassfile'],
 800             '-keypass:file', config['keypassfile'],
 801             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 802             os.path.join(repodir, 'index.jar') , config['repo_keyalias']])
 803         # TODO keypass should be sent via stdin
 804         if p.returncode != 0:
 805             logging.info("Failed to sign index")
 806             sys.exit(1)
 807
 808     # Copy the repo icon into the repo directory...
 809     icon_dir = os.path.join(repodir ,'icons')
 810     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 811     shutil.copyfile(config['repo_icon'], iconfilename)
 812
 813     # Write a category list in the repo to allow quick access...
 814     catdata = ''
 815     for cat in categories:
 816         catdata += cat + '\n'
 817     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 818     f.write(catdata)
 819     f.close()
 820
 821
 822
 823 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 824
 825     for app in apps:
 826
 827         # Get a list of the apks for this app...
 828         apklist = []
 829         for apk in apks:
 830             if apk['id'] == app['id']:
 831                 apklist.append(apk)
 832
 833         # Sort the apk list into version order...
 834         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 835
 836         if app['Archive Policy']:
 837             keepversions = int(app['Archive Policy'][:-9])
 838         else:
 839             keepversions = defaultkeepversions
 840
 841         if len(apklist) > keepversions:
 842             for apk in apklist[keepversions:]:
 843                 logging.info("Moving " + apk['apkname'] + " to archive")
 844                 shutil.move(os.path.join(repodir, apk['apkname']),
 845                     os.path.join(archivedir, apk['apkname']))
 846                 if 'srcname' in apk:
 847                     shutil.move(os.path.join(repodir, apk['srcname']),
 848                         os.path.join(archivedir, apk['srcname']))
 849                 archapks.append(apk)
 850                 apks.remove(apk)
 851
 852
 853 config = None
 854 options = None
 855
 856 def main():
 857
 858     global config, options
 859
 860     # Parse command line...
 861     parser = OptionParser()
 862     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 863                       help="Create skeleton metadata files that are missing")
 864     parser.add_option("-v", "--verbose", action="store_true", default=False,
 865                       help="Spew out even more information than normal")
 866     parser.add_option("-q", "--quiet", action="store_true", default=False,
 867                       help="Restrict output to warnings and errors")
 868     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 869                       help="Report on build data status")
 870     parser.add_option("-i", "--interactive", default=False, action="store_true",
 871                       help="Interactively ask about things that need updating.")
 872     parser.add_option("-I", "--icons", action="store_true", default=False,
 873                       help="Resize all the icons exceeding the max pixel size and exit")
 874     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 875                       help="Specify editor to use in interactive mode. Default "+
 876                           "is /etc/alternatives/editor")
 877     parser.add_option("-w", "--wiki", default=False, action="store_true",
 878                       help="Update the wiki")
 879     parser.add_option("", "--pretty", action="store_true", default=False,
 880                       help="Produce human-readable index.xml")
 881     parser.add_option("--clean", action="store_true", default=False,
 882                       help="Clean update - don't uses caches, reprocess all apks")
 883     (options, args) = parser.parse_args()
 884
 885     config = common.read_config(options)
 886
 887     repodirs = ['repo']
 888     if config['archive_older'] != 0:
 889         repodirs.append('archive')
 890         if not os.path.exists('archive'):
 891             os.mkdir('archive')
 892
 893     if options.icons:
 894         resize_all_icons(repodirs)
 895         sys.exit(0)
 896
 897     # Get all apps...
 898     apps = metadata.read_metadata()
 899
 900     # Generate a list of categories...
 901     categories = set()
 902     for app in apps:
 903         categories.update(app['Categories'])
 904
 905     # Read known apks data (will be updated and written back when we've finished)
 906     knownapks = common.KnownApks()
 907
 908     # Gather information about all the apk files in the repo directory, using
 909     # cached data if possible.
 910     apkcachefile = os.path.join('tmp', 'apkcache')
 911     if not options.clean and os.path.exists(apkcachefile):
 912         with open(apkcachefile, 'rb') as cf:
 913             apkcache = pickle.load(cf)
 914     else:
 915         apkcache = {}
 916     cachechanged = False
 917
 918     delete_disabled_builds(apps, apkcache, repodirs)
 919
 920     # Scan all apks in the main repo
 921     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 922     if cc:
 923         cachechanged = True
 924
 925     # Scan the archive repo for apks as well
 926     if len(repodirs) > 1:
 927         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 928         if cc:
 929             cachechanged = True
 930     else:
 931         archapks = []
 932
 933     # Some information from the apks needs to be applied up to the application
 934     # level. When doing this, we use the info from the most recent version's apk.
 935     # We deal with figuring out when the app was added and last updated at the
 936     # same time.
 937     for app in apps:
 938         bestver = 0
 939         added = None
 940         lastupdated = None
 941         for apk in apks + archapks:
 942             if apk['id'] == app['id']:
 943                 if apk['versioncode'] > bestver:
 944                     bestver = apk['versioncode']
 945                     bestapk = apk
 946
 947                 if 'added' in apk:
 948                     if not added or apk['added'] < added:
 949                         added = apk['added']
 950                     if not lastupdated or apk['added'] > lastupdated:
 951                         lastupdated = apk['added']
 952
 953         if added:
 954             app['added'] = added
 955         else:
 956             logging.warn("Don't know when " + app['id'] + " was added")
 957         if lastupdated:
 958             app['lastupdated'] = lastupdated
 959         else:
 960             logging.warn("Don't know when " + app['id'] + " was last updated")
 961
 962         if bestver == 0:
 963             if app['Name'] is None:
 964                 app['Name'] = app['id']
 965             app['icon'] = None
 966             logging.warn("Application " + app['id'] + " has no packages")
 967         else:
 968             if app['Name'] is None:
 969                 app['Name'] = bestapk['name']
 970             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 971
 972     # Sort the app list by name, then the web site doesn't have to by default.
 973     # (we had to wait until we'd scanned the apks to do this, because mostly the
 974     # name comes from there!)
 975     apps = sorted(apps, key=lambda app: app['Name'].upper())
 976
 977     # Generate warnings for apk's with no metadata (or create skeleton
 978     # metadata files, if requested on the command line)
 979     for apk in apks:
 980         found = False
 981         for app in apps:
 982             if app['id'] == apk['id']:
 983                 found = True
 984                 break
 985         if not found:
 986             if options.createmeta:
 987                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 988                 f.write("License:Unknown\n")
 989                 f.write("Web Site:\n")
 990                 f.write("Source Code:\n")
 991                 f.write("Issue Tracker:\n")
 992                 f.write("Summary:" + apk['name'] + "\n")
 993                 f.write("Description:\n")
 994                 f.write(apk['name'] + "\n")
 995                 f.write(".\n")
 996                 f.close()
 997                 logging.info("Generated skeleton metadata for " + apk['id'])
 998             else:
 999                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata - removing")
1000                 rmf = os.path.join(repodirs[0], apk['apkname'])
1001                 if not os.path.exists(rmf):
1002                     logging.error("Could not find {0} to remove it".format(rmf))
1003                 else:
1004                     os.remove(rmf)
1005
1006     if len(repodirs) > 1:
1007         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1008
1009     # Make the index for the main repo...
1010     make_index(apps, apks, repodirs[0], False, categories)
1011
1012     # If there's an archive repo,  make the index for it. We already scanned it
1013     # earlier on.
1014     if len(repodirs) > 1:
1015         make_index(apps, archapks, repodirs[1], True, categories)
1016
1017     if config['update_stats']:
1018
1019         # Update known apks info...
1020         knownapks.writeifchanged()
1021
1022         # Generate latest apps data for widget
1023         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1024             data = ''
1025             for line in file(os.path.join('stats', 'latestapps.txt')):
1026                 appid = line.rstrip()
1027                 data += appid + "\t"
1028                 for app in apps:
1029                     if app['id'] == appid:
1030                         data += app['Name'] + "\t"
1031                         if app['icon'] is not None:
1032                             data += app['icon'] + "\t"
1033                         data += app['License'] + "\n"
1034                         break
1035             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1036             f.write(data)
1037             f.close()
1038
1039     if cachechanged:
1040         with open(apkcachefile, 'wb') as cf:
1041             pickle.dump(apkcache, cf)
1042
1043     # Update the wiki...
1044     if options.wiki:
1045         update_wiki(apps, apks + archapks)
1046
1047     logging.info("Finished.")
1048
1049 if __name__ == "__main__":
1050     main()
1051