fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         if app['Current Version Code'] == '0':
 145             cantupdate = True
 146         # Sort with most recent first...
 147         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 148
 149         wikidata += "=Versions=\n"
 150         if len(apklist) == 0:
 151             wikidata += "We currently have no versions of this app available."
 152         elif not gotcurrentver:
 153             wikidata += "We don't have the current version of this app."
 154         else:
 155             wikidata += "We have the current version of this app."
 156         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 157         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 158         if len(app['No Source Since']) > 0:
 159             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 160         if len(app['Current Version']) > 0:
 161             wikidata += "The current (recommended) version is " + app['Current Version']
 162             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 163         validapks = 0
 164         for apk in apklist:
 165             wikidata += "==" + apk['version'] + "==\n"
 166
 167             if 'buildproblem' in apk:
 168                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 169             else:
 170                 validapks += 1
 171                 wikidata += "This version is built and signed by "
 172                 if 'srcname' in apk:
 173                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 174                 else:
 175                     wikidata += "the original developer.\n\n"
 176             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 177
 178         wikidata += '\n[[Category:' + wikicat + ']]\n'
 179         if len(app['No Source Since']) > 0:
 180             wikidata += '\n[[Category:Apps missing source code]]\n'
 181         if validapks == 0 and not app['Disabled']:
 182             wikidata += '\n[[Category:Apps with no packages]]\n'
 183         if cantupdate and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps we can't update]]\n"
 185         if buildfails and not app['Disabled']:
 186             wikidata += "\n[[Category:Apps with failing builds]]\n"
 187         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 188             wikidata += '\n[[Category:Apps to Update]]\n'
 189         if app['Disabled']:
 190             wikidata += '\n[[Category:Apps that are disabled]]\n'
 191         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 192             wikidata += '\n[[Category:Apps with no update check]]\n'
 193         for appcat in app['Categories']:
 194             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 195
 196         # We can't have underscores in the page name, even if they're in
 197         # the package ID, because MediaWiki messes with them...
 198         pagename = app['id'].replace('_', ' ')
 199
 200         # Drop a trailing newline, because mediawiki is going to drop it anyway
 201         # and it we don't we'll think the page has changed when it hasn't...
 202         if wikidata.endswith('\n'):
 203             wikidata = wikidata[:-1]
 204
 205         generated_pages[pagename] = wikidata
 206
 207         # Make a redirect from the name to the ID too, unless there's
 208         # already an existing page with the name and it isn't a redirect.
 209         noclobber = False
 210         apppagename = app['Name'].replace('_', ' ')
 211         apppagename = apppagename.replace('{', '')
 212         apppagename = apppagename.replace('}', ' ')
 213         apppagename = apppagename.replace(':', ' ')
 214         # Drop double spaces caused mostly by replacing ':' above
 215         apppagename = apppagename.replace('  ', ' ')
 216         for expagename in site.allpages(prefix=apppagename,
 217                 filterredir='nonredirects', generator=False):
 218             if expagename == apppagename:
 219                 noclobber = True
 220         # Another reason not to make the redirect page is if the app name
 221         # is the same as it's ID, because that will overwrite the real page
 222         # with an redirect to itself! (Although it seems like an odd
 223         # scenario this happens a lot, e.g. where there is metadata but no
 224         # builds or binaries to extract a name from.
 225         if apppagename == pagename:
 226             noclobber = True
 227         if not noclobber:
 228             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 229
 230     for tcat, genp in [(wikicat, generated_pages),
 231             (wikiredircat, generated_redirects)]:
 232         catpages = site.Pages['Category:' + tcat]
 233         existingpages = []
 234         for page in catpages:
 235             existingpages.append(page.name)
 236             if page.name in genp:
 237                 pagetxt = page.edit()
 238                 if pagetxt != genp[page.name]:
 239                     logging.debug("Updating modified page " + page.name)
 240                     page.save(genp[page.name], summary='Auto-updated')
 241                 else:
 242                     logging.debug("Page " + page.name + " is unchanged")
 243             else:
 244                 logging.warn("Deleting page " + page.name)
 245                 page.delete('No longer published')
 246         for pagename, text in genp.items():
 247             logging.debug("Checking " + pagename)
 248             if not pagename in existingpages:
 249                 logging.debug("Creating page " + pagename)
 250                 try:
 251                     newpage = site.Pages[pagename]
 252                     newpage.save(text, summary='Auto-created')
 253                 except:
 254                     logging.error("...FAILED to create page")
 255
 256     # Purge server cache to ensure counts are up to date
 257     site.pages['Repository Maintenance'].purge()
 258
 259 def delete_disabled_builds(apps, apkcache, repodirs):
 260     """Delete disabled build outputs.
 261
 262     :param apps: list of all applications, as per metadata.read_metadata
 263     :param apkcache: current apk cache information
 264     :param repodirs: the repo directories to process
 265     """
 266     for app in apps:
 267         for build in app['builds']:
 268             if 'disable' in build:
 269                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 270                 for repodir in repodirs:
 271                     apkpath = os.path.join(repodir, apkfilename)
 272                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 273                     for name in [apkpath, srcpath]:
 274                         if os.path.exists(name):
 275                             logging.warn("Deleting disabled build output " + apkfilename)
 276                             os.remove(name)
 277                 if apkfilename in apkcache:
 278                     del apkcache[apkfilename]
 279
 280 def resize_icon(iconpath, density):
 281
 282     if not os.path.isfile(iconpath):
 283         return
 284
 285     try:
 286         im = Image.open(iconpath)
 287         size = dpi_to_px(density)
 288
 289         if any(length > size for length in im.size):
 290             oldsize = im.size
 291             im.thumbnail((size, size), Image.ANTIALIAS)
 292             logging.debug("%s was too large at %s - new size is %s" % (
 293                 iconpath, oldsize, im.size))
 294             im.save(iconpath, "PNG")
 295
 296         else:
 297             logging.info("%s is small enough: %s" % im.size)
 298
 299     except Exception,e:
 300         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 301
 302 def resize_all_icons(repodirs):
 303     """Resize all icons that exceed the max size
 304
 305     :param repodirs: the repo directories to process
 306     """
 307     for repodir in repodirs:
 308         for density in get_densities():
 309             icon_dir = get_icon_dir(repodir, density)
 310             icon_glob = os.path.join(icon_dir, '*.png')
 311             for iconpath in glob.glob(icon_glob):
 312                 resize_icon(iconpath, density)
 313
 314 def scan_apks(apps, apkcache, repodir, knownapks):
 315     """Scan the apks in the given repo directory.
 316
 317     This also extracts the icons.
 318
 319     :param apps: list of all applications, as per metadata.read_metadata
 320     :param apkcache: current apk cache information
 321     :param repodir: repo directory to scan
 322     :param knownapks: known apks info
 323     :returns: (apks, cachechanged) where apks is a list of apk information,
 324               and cachechanged is True if the apkcache got changed.
 325     """
 326
 327     cachechanged = False
 328
 329     icon_dirs = get_icon_dirs(repodir)
 330     for icon_dir in icon_dirs:
 331         if os.path.exists(icon_dir):
 332             if options.clean:
 333                 shutil.rmtree(icon_dir)
 334                 os.makedirs(icon_dir)
 335         else:
 336             os.makedirs(icon_dir)
 337
 338     apks = []
 339     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 340     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 341     vername_pat = re.compile(".*versionName='([^']*)'.*")
 342     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 343     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 344     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 345     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 346     string_pat = re.compile(".*'([^']*)'.*")
 347     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 348
 349         apkfilename = apkfile[len(repodir) + 1:]
 350         if ' ' in apkfilename:
 351             logging.error("No spaces in APK filenames!")
 352             sys.exit(1)
 353
 354         if apkfilename in apkcache:
 355             logging.debug("Reading " + apkfilename + " from cache")
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             logging.info("Processing " + apkfilename)
 361             thisinfo = {}
 362             thisinfo['apkname'] = apkfilename
 363             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 364             if os.path.exists(os.path.join(repodir, srcfilename)):
 365                 thisinfo['srcname'] = srcfilename
 366             thisinfo['size'] = os.path.getsize(apkfile)
 367             thisinfo['permissions'] = []
 368             thisinfo['features'] = []
 369             thisinfo['icons_src'] = {}
 370             thisinfo['icons'] = {}
 371             p = FDroidPopen([os.path.join(config['sdk_path'],
 372                         'build-tools', config['build_tools'], 'aapt'),
 373                     'dump', 'badging', apkfile])
 374             if p.returncode != 0:
 375                 logging.critical("Failed to get apk information")
 376                 sys.exit(1)
 377             for line in p.stdout.splitlines():
 378                 if line.startswith("package:"):
 379                     try:
 380                         thisinfo['id'] = re.match(name_pat, line).group(1)
 381                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 382                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 383                     except Exception, e:
 384                         logging.info("Package matching failed: " + str(e))
 385                         logging.info("Line was: " + line)
 386                         sys.exit(1)
 387                 elif line.startswith("application:"):
 388                     thisinfo['name'] = re.match(label_pat, line).group(1)
 389                     # Keep path to non-dpi icon in case we need it
 390                     match = re.match(icon_pat_nodpi, line)
 391                     if match:
 392                         thisinfo['icons_src']['-1'] = match.group(1)
 393                 elif line.startswith("launchable-activity:"):
 394                     # Only use launchable-activity as fallback to application
 395                     if '-1' not in thisinfo['icons_src']:
 396                         match = re.match(icon_pat_nodpi, line)
 397                         if match:
 398                             thisinfo['icons_src']['-1'] = match.group(1)
 399                 elif line.startswith("application-icon-"):
 400                     match = re.match(icon_pat, line)
 401                     if match:
 402                         density = match.group(1)
 403                         path = match.group(2)
 404                         thisinfo['icons_src'][density] = path
 405                 elif line.startswith("sdkVersion:"):
 406                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 407                 elif line.startswith("maxSdkVersion:"):
 408                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 logging.warn("no SDK version information found")
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 450                 sys.exit(1)
 451             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 452                         'getsig', os.path.join(os.getcwd(), apkfile)])
 453             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 454                 logging.critical("Failed to get apk signature")
 455                 sys.exit(1)
 456             thisinfo['sig'] = p.stdout[7:].strip()
 457
 458             apk = zipfile.ZipFile(apkfile, 'r')
 459
 460             iconfilename = "%s.%s.png" % (
 461                     thisinfo['id'],
 462                     thisinfo['versioncode'])
 463
 464             # Extract the icon file...
 465             densities = get_densities()
 466             empty_densities = []
 467             for density in densities:
 468                 if density not in thisinfo['icons_src']:
 469                     empty_densities.append(density)
 470                     continue
 471                 iconsrc = thisinfo['icons_src'][density]
 472                 icon_dir = get_icon_dir(repodir, density)
 473                 icondest = os.path.join(icon_dir, iconfilename)
 474
 475                 try:
 476                     iconfile = open(icondest, 'wb')
 477                     iconfile.write(apk.read(iconsrc))
 478                     iconfile.close()
 479                     thisinfo['icons'][density] = iconfilename
 480
 481                 except:
 482                     logging.warn("Error retrieving icon file")
 483                     del thisinfo['icons'][density]
 484                     del thisinfo['icons_src'][density]
 485                     empty_densities.append(density)
 486
 487             if '-1' in thisinfo['icons_src']:
 488                 iconsrc = thisinfo['icons_src']['-1']
 489                 iconpath = os.path.join(
 490                         get_icon_dir(repodir, None), iconfilename)
 491                 iconfile = open(iconpath, 'wb')
 492                 iconfile.write(apk.read(iconsrc))
 493                 iconfile.close()
 494                 try:
 495                     im = Image.open(iconpath)
 496                     dpi = px_to_dpi(im.size[0])
 497                     for density in densities:
 498                         if density in thisinfo['icons']:
 499                             break
 500                         if density == densities[-1] or dpi >= int(density):
 501                             thisinfo['icons'][density] = iconfilename
 502                             shutil.move(iconpath,
 503                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 504                             empty_densities.remove(density)
 505                             break
 506                 except Exception,e:
 507                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 508
 509             if thisinfo['icons']:
 510                 thisinfo['icon'] = iconfilename
 511
 512             apk.close()
 513
 514             # First try resizing down to not lose quality
 515             last_density = None
 516             for density in densities:
 517                 if density not in empty_densities:
 518                     last_density = density
 519                     continue
 520                 if last_density is None:
 521                     continue
 522                 logging.info("Density %s not available, resizing down from %s" % (
 523                         density, last_density))
 524
 525                 last_iconpath = os.path.join(
 526                         get_icon_dir(repodir, last_density), iconfilename)
 527                 iconpath = os.path.join(
 528                         get_icon_dir(repodir, density), iconfilename)
 529                 try:
 530                     im = Image.open(last_iconpath)
 531                 except:
 532                     logging.warn("Invalid image file at %s" % last_iconpath)
 533                     continue
 534
 535                 size = dpi_to_px(density)
 536
 537                 im.thumbnail((size, size), Image.ANTIALIAS)
 538                 im.save(iconpath, "PNG")
 539                 empty_densities.remove(density)
 540
 541             # Then just copy from the highest resolution available
 542             last_density = None
 543             for density in reversed(densities):
 544                 if density not in empty_densities:
 545                     last_density = density
 546                     continue
 547                 if last_density is None:
 548                     continue
 549                 logging.info("Density %s not available, copying from lower density %s" % (
 550                         density, last_density))
 551
 552                 shutil.copyfile(
 553                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 554                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 555
 556                 empty_densities.remove(density)
 557
 558             for density in densities:
 559                 icon_dir = get_icon_dir(repodir, density)
 560                 icondest = os.path.join(icon_dir, iconfilename)
 561                 resize_icon(icondest, density)
 562
 563             # Copy from icons-mdpi to icons since mdpi is the baseline density
 564             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 565             if os.path.isfile(baseline):
 566                 shutil.copyfile(baseline,
 567                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 568
 569             # Record in known apks, getting the added date at the same time..
 570             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 571             if added:
 572                 thisinfo['added'] = added
 573
 574             apkcache[apkfilename] = thisinfo
 575             cachechanged = True
 576
 577         apks.append(thisinfo)
 578
 579     return apks, cachechanged
 580
 581
 582 repo_pubkey_fingerprint = None
 583
 584 def make_index(apps, apks, repodir, archive, categories):
 585     """Make a repo index.
 586
 587     :param apps: fully populated apps list
 588     :param apks: full populated apks list
 589     :param repodir: the repo directory
 590     :param archive: True if this is the archive repo, False if it's the
 591                     main one.
 592     :param categories: list of categories
 593     """
 594
 595     doc = Document()
 596
 597     def addElement(name, value, doc, parent):
 598         el = doc.createElement(name)
 599         el.appendChild(doc.createTextNode(value))
 600         parent.appendChild(el)
 601     def addElementCDATA(name, value, doc, parent):
 602         el = doc.createElement(name)
 603         el.appendChild(doc.createCDATASection(value))
 604         parent.appendChild(el)
 605
 606     root = doc.createElement("fdroid")
 607     doc.appendChild(root)
 608
 609     repoel = doc.createElement("repo")
 610
 611     if archive:
 612         repoel.setAttribute("name", config['archive_name'])
 613         if config['repo_maxage'] != 0:
 614             repoel.setAttribute("maxage", str(config['repo_maxage']))
 615         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 616         repoel.setAttribute("url", config['archive_url'])
 617         addElement('description', config['archive_description'], doc, repoel)
 618
 619     else:
 620         repoel.setAttribute("name", config['repo_name'])
 621         if config['repo_maxage'] != 0:
 622             repoel.setAttribute("maxage", str(config['repo_maxage']))
 623         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 624         repoel.setAttribute("url", config['repo_url'])
 625         addElement('description', config['repo_description'], doc, repoel)
 626
 627     repoel.setAttribute("version", "12")
 628     repoel.setAttribute("timestamp", str(int(time.time())))
 629
 630     if config['repo_keyalias']:
 631
 632         # Generate a certificate fingerprint the same way keytool does it
 633         # (but with slightly different formatting)
 634         def cert_fingerprint(data):
 635             digest = hashlib.sha1(data).digest()
 636             ret = []
 637             for i in range(4):
 638                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 639             return " ".join(ret)
 640
 641         def extract_pubkey():
 642             p = FDroidPopen(['keytool', '-exportcert',
 643                                   '-alias', config['repo_keyalias'],
 644                                   '-keystore', config['keystore'],
 645                                   '-storepass', config['keystorepass']])
 646             if p.returncode != 0:
 647                 logging.critical("Failed to get repo pubkey")
 648                 sys.exit(1)
 649             global repo_pubkey_fingerprint
 650             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 651             return "".join("%02x" % ord(b) for b in p.stdout)
 652
 653         repoel.setAttribute("pubkey", extract_pubkey())
 654
 655     root.appendChild(repoel)
 656
 657     for app in apps:
 658
 659         if app['Disabled'] is not None:
 660             continue
 661
 662         # Get a list of the apks for this app...
 663         apklist = []
 664         for apk in apks:
 665             if apk['id'] == app['id']:
 666                 apklist.append(apk)
 667
 668         if len(apklist) == 0:
 669             continue
 670
 671         apel = doc.createElement("application")
 672         apel.setAttribute("id", app['id'])
 673         root.appendChild(apel)
 674
 675         addElement('id', app['id'], doc, apel)
 676         if 'added' in app:
 677             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 678         if 'lastupdated' in app:
 679             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 680         addElement('name', app['Name'], doc, apel)
 681         addElement('summary', app['Summary'], doc, apel)
 682         if app['icon']:
 683             addElement('icon', app['icon'], doc, apel)
 684         def linkres(link):
 685             for app in apps:
 686                 if app['id'] == link:
 687                     return ("fdroid.app:" + link, app['Name'])
 688             raise MetaDataException("Cannot resolve app id " + link)
 689         addElement('desc',
 690                 metadata.description_html(app['Description'], linkres), doc, apel)
 691         addElement('license', app['License'], doc, apel)
 692         if 'Categories' in app:
 693             addElement('categories', ','.join(app["Categories"]), doc, apel)
 694             # We put the first (primary) category in LAST, which will have
 695             # the desired effect of making clients that only understand one
 696             # category see that one.
 697             addElement('category', app["Categories"][0], doc, apel)
 698         addElement('web', app['Web Site'], doc, apel)
 699         addElement('source', app['Source Code'], doc, apel)
 700         addElement('tracker', app['Issue Tracker'], doc, apel)
 701         if app['Donate']:
 702             addElement('donate', app['Donate'], doc, apel)
 703         if app['Bitcoin']:
 704             addElement('bitcoin', app['Bitcoin'], doc, apel)
 705         if app['Litecoin']:
 706             addElement('litecoin', app['Litecoin'], doc, apel)
 707         if app['Dogecoin']:
 708             addElement('dogecoin', app['Dogecoin'], doc, apel)
 709         if app['FlattrID']:
 710             addElement('flattr', app['FlattrID'], doc, apel)
 711
 712         # These elements actually refer to the current version (i.e. which
 713         # one is recommended. They are historically mis-named, and need
 714         # changing, but stay like this for now to support existing clients.
 715         addElement('marketversion', app['Current Version'], doc, apel)
 716         addElement('marketvercode', app['Current Version Code'], doc, apel)
 717
 718         if app['AntiFeatures']:
 719             af = app['AntiFeatures'].split(',')
 720             # TODO: Temporarily not including UpstreamNonFree in the index,
 721             # because current F-Droid clients do not understand it, and also
 722             # look ugly when they encounter an unknown antifeature. This
 723             # filtering can be removed in time...
 724             if 'UpstreamNonFree' in af:
 725                 af.remove('UpstreamNonFree')
 726             if af:
 727                 addElement('antifeatures', ','.join(af), doc, apel)
 728         if app['Provides']:
 729             pv = app['Provides'].split(',')
 730             addElement('provides', ','.join(pv), doc, apel)
 731         if app['Requires Root']:
 732             addElement('requirements', 'root', doc, apel)
 733
 734         # Sort the apk list into version order, just so the web site
 735         # doesn't have to do any work by default...
 736         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 737
 738         # Check for duplicates - they will make the client unhappy...
 739         for i in range(len(apklist) - 1):
 740             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 741                 logging.critical("duplicate versions: '%s' - '%s'" % (
 742                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 743                 sys.exit(1)
 744
 745         for apk in apklist:
 746             apkel = doc.createElement("package")
 747             apel.appendChild(apkel)
 748             addElement('version', apk['version'], doc, apkel)
 749             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 750             addElement('apkname', apk['apkname'], doc, apkel)
 751             if 'srcname' in apk:
 752                 addElement('srcname', apk['srcname'], doc, apkel)
 753             for hash_type in ['sha256']:
 754                 if not hash_type in apk:
 755                     continue
 756                 hashel = doc.createElement("hash")
 757                 hashel.setAttribute("type", hash_type)
 758                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 759                 apkel.appendChild(hashel)
 760             addElement('sig', apk['sig'], doc, apkel)
 761             addElement('size', str(apk['size']), doc, apkel)
 762             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 763             if 'maxsdkversion' in apk:
 764                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 765             if 'added' in apk:
 766                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 767             if app['Requires Root']:
 768                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 769                     apk['permissions'].append('ACCESS_SUPERUSER')
 770
 771             if len(apk['permissions']) > 0:
 772                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 773             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 774                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 775             if len(apk['features']) > 0:
 776                 addElement('features', ','.join(apk['features']), doc, apkel)
 777
 778     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 779     if options.pretty:
 780         output = doc.toprettyxml()
 781     else:
 782         output = doc.toxml()
 783     of.write(output)
 784     of.close()
 785
 786     if config['repo_keyalias'] is not None:
 787
 788         logging.info("Creating signed index.")
 789         logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint)
 790
 791         #Create a jar of the index...
 792         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 793         if p.returncode != 0:
 794             logging.critical("Failed to create jar file")
 795             sys.exit(1)
 796
 797         # Sign the index...
 798         p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
 799             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 800             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 801             os.path.join(repodir, 'index.jar') , config['repo_keyalias']])
 802         if p.returncode != 0:
 803             logging.info("Failed to sign index")
 804             sys.exit(1)
 805
 806     # Copy the repo icon into the repo directory...
 807     icon_dir = os.path.join(repodir ,'icons')
 808     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 809     shutil.copyfile(config['repo_icon'], iconfilename)
 810
 811     # Write a category list in the repo to allow quick access...
 812     catdata = ''
 813     for cat in categories:
 814         catdata += cat + '\n'
 815     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 816     f.write(catdata)
 817     f.close()
 818
 819
 820
 821 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 822
 823     for app in apps:
 824
 825         # Get a list of the apks for this app...
 826         apklist = []
 827         for apk in apks:
 828             if apk['id'] == app['id']:
 829                 apklist.append(apk)
 830
 831         # Sort the apk list into version order...
 832         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 833
 834         if app['Archive Policy']:
 835             keepversions = int(app['Archive Policy'][:-9])
 836         else:
 837             keepversions = defaultkeepversions
 838
 839         if len(apklist) > keepversions:
 840             for apk in apklist[keepversions:]:
 841                 logging.info("Moving " + apk['apkname'] + " to archive")
 842                 shutil.move(os.path.join(repodir, apk['apkname']),
 843                     os.path.join(archivedir, apk['apkname']))
 844                 if 'srcname' in apk:
 845                     shutil.move(os.path.join(repodir, apk['srcname']),
 846                         os.path.join(archivedir, apk['srcname']))
 847                 archapks.append(apk)
 848                 apks.remove(apk)
 849
 850
 851 config = None
 852 options = None
 853
 854 def main():
 855
 856     global config, options
 857
 858     # Parse command line...
 859     parser = OptionParser()
 860     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 861                       help="Create skeleton metadata files that are missing")
 862     parser.add_option("-v", "--verbose", action="store_true", default=False,
 863                       help="Spew out even more information than normal")
 864     parser.add_option("-q", "--quiet", action="store_true", default=False,
 865                       help="Restrict output to warnings and errors")
 866     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 867                       help="Report on build data status")
 868     parser.add_option("-i", "--interactive", default=False, action="store_true",
 869                       help="Interactively ask about things that need updating.")
 870     parser.add_option("-I", "--icons", action="store_true", default=False,
 871                       help="Resize all the icons exceeding the max pixel size and exit")
 872     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 873                       help="Specify editor to use in interactive mode. Default "+
 874                           "is /etc/alternatives/editor")
 875     parser.add_option("-w", "--wiki", default=False, action="store_true",
 876                       help="Update the wiki")
 877     parser.add_option("", "--pretty", action="store_true", default=False,
 878                       help="Produce human-readable index.xml")
 879     parser.add_option("--clean", action="store_true", default=False,
 880                       help="Clean update - don't uses caches, reprocess all apks")
 881     (options, args) = parser.parse_args()
 882
 883     config = common.read_config(options)
 884
 885     repodirs = ['repo']
 886     if config['archive_older'] != 0:
 887         repodirs.append('archive')
 888         if not os.path.exists('archive'):
 889             os.mkdir('archive')
 890
 891     if options.icons:
 892         resize_all_icons(repodirs)
 893         sys.exit(0)
 894
 895     # Get all apps...
 896     apps = metadata.read_metadata()
 897
 898     # Generate a list of categories...
 899     categories = set()
 900     for app in apps:
 901         categories.update(app['Categories'])
 902
 903     # Read known apks data (will be updated and written back when we've finished)
 904     knownapks = common.KnownApks()
 905
 906     # Gather information about all the apk files in the repo directory, using
 907     # cached data if possible.
 908     apkcachefile = os.path.join('tmp', 'apkcache')
 909     if not options.clean and os.path.exists(apkcachefile):
 910         with open(apkcachefile, 'rb') as cf:
 911             apkcache = pickle.load(cf)
 912     else:
 913         apkcache = {}
 914     cachechanged = False
 915
 916     delete_disabled_builds(apps, apkcache, repodirs)
 917
 918     # Scan all apks in the main repo
 919     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 920     if cc:
 921         cachechanged = True
 922
 923     # Scan the archive repo for apks as well
 924     if len(repodirs) > 1:
 925         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 926         if cc:
 927             cachechanged = True
 928     else:
 929         archapks = []
 930
 931     # Some information from the apks needs to be applied up to the application
 932     # level. When doing this, we use the info from the most recent version's apk.
 933     # We deal with figuring out when the app was added and last updated at the
 934     # same time.
 935     for app in apps:
 936         bestver = 0
 937         added = None
 938         lastupdated = None
 939         for apk in apks + archapks:
 940             if apk['id'] == app['id']:
 941                 if apk['versioncode'] > bestver:
 942                     bestver = apk['versioncode']
 943                     bestapk = apk
 944
 945                 if 'added' in apk:
 946                     if not added or apk['added'] < added:
 947                         added = apk['added']
 948                     if not lastupdated or apk['added'] > lastupdated:
 949                         lastupdated = apk['added']
 950
 951         if added:
 952             app['added'] = added
 953         else:
 954             logging.warn("Don't know when " + app['id'] + " was added")
 955         if lastupdated:
 956             app['lastupdated'] = lastupdated
 957         else:
 958             logging.warn("Don't know when " + app['id'] + " was last updated")
 959
 960         if bestver == 0:
 961             if app['Name'] is None:
 962                 app['Name'] = app['id']
 963             app['icon'] = None
 964             logging.warn("Application " + app['id'] + " has no packages")
 965         else:
 966             if app['Name'] is None:
 967                 app['Name'] = bestapk['name']
 968             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 969
 970     # Sort the app list by name, then the web site doesn't have to by default.
 971     # (we had to wait until we'd scanned the apks to do this, because mostly the
 972     # name comes from there!)
 973     apps = sorted(apps, key=lambda app: app['Name'].upper())
 974
 975     # Generate warnings for apk's with no metadata (or create skeleton
 976     # metadata files, if requested on the command line)
 977     for apk in apks:
 978         found = False
 979         for app in apps:
 980             if app['id'] == apk['id']:
 981                 found = True
 982                 break
 983         if not found:
 984             if options.createmeta:
 985                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 986                 f.write("License:Unknown\n")
 987                 f.write("Web Site:\n")
 988                 f.write("Source Code:\n")
 989                 f.write("Issue Tracker:\n")
 990                 f.write("Summary:" + apk['name'] + "\n")
 991                 f.write("Description:\n")
 992                 f.write(apk['name'] + "\n")
 993                 f.write(".\n")
 994                 f.close()
 995                 logging.info("Generated skeleton metadata for " + apk['id'])
 996             else:
 997                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata")
 998                 logging.info("       " + apk['name'] + " - " + apk['version'])
 999
1000     if len(repodirs) > 1:
1001         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1002
1003     # Make the index for the main repo...
1004     make_index(apps, apks, repodirs[0], False, categories)
1005
1006     # If there's an archive repo,  make the index for it. We already scanned it
1007     # earlier on.
1008     if len(repodirs) > 1:
1009         make_index(apps, archapks, repodirs[1], True, categories)
1010
1011     if config['update_stats']:
1012
1013         # Update known apks info...
1014         knownapks.writeifchanged()
1015
1016         # Generate latest apps data for widget
1017         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1018             data = ''
1019             for line in file(os.path.join('stats', 'latestapps.txt')):
1020                 appid = line.rstrip()
1021                 data += appid + "\t"
1022                 for app in apps:
1023                     if app['id'] == appid:
1024                         data += app['Name'] + "\t"
1025                         if app['icon'] is not None:
1026                             data += app['icon'] + "\t"
1027                         data += app['License'] + "\n"
1028                         break
1029             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1030             f.write(data)
1031             f.close()
1032
1033     if cachechanged:
1034         with open(apkcachefile, 'wb') as cf:
1035             pickle.dump(apkcache, cf)
1036
1037     # Update the wiki...
1038     if options.wiki:
1039         update_wiki(apps, apks + archapks)
1040
1041     logging.info("Finished.")
1042
1043 if __name__ == "__main__":
1044     main()
1045