fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def dpi_to_px(density):
  42     return (int(density) * 48) / 160
  43
  44 def px_to_dpi(px):
  45     return (int(px) * 160) / 48
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         if validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         if cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         if buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 191             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 192
 193         # We can't have underscores in the page name, even if they're in
 194         # the package ID, because MediaWiki messes with them...
 195         pagename = app['id'].replace('_', ' ')
 196
 197         # Drop a trailing newline, because mediawiki is going to drop it anyway
 198         # and it we don't we'll think the page has changed when it hasn't...
 199         if wikidata.endswith('\n'):
 200             wikidata = wikidata[:-1]
 201
 202         generated_pages[pagename] = wikidata
 203
 204         # Make a redirect from the name to the ID too, unless there's
 205         # already an existing page with the name and it isn't a redirect.
 206         noclobber = False
 207         apppagename = app['Name'].replace('_', ' ')
 208         apppagename = apppagename.replace('{', '')
 209         apppagename = apppagename.replace('}', ' ')
 210         apppagename = apppagename.replace(':', ' ')
 211         # Drop double spaces caused mostly by replacing ':' above
 212         apppagename = apppagename.replace('  ', ' ')
 213         for expagename in site.allpages(prefix=apppagename,
 214                 filterredir='nonredirects', generator=False):
 215             if expagename == apppagename:
 216                 noclobber = True
 217         # Another reason not to make the redirect page is if the app name
 218         # is the same as it's ID, because that will overwrite the real page
 219         # with an redirect to itself! (Although it seems like an odd
 220         # scenario this happens a lot, e.g. where there is metadata but no
 221         # builds or binaries to extract a name from.
 222         if apppagename == pagename:
 223             noclobber = True
 224         if not noclobber:
 225             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 226
 227     for tcat, genp in [(wikicat, generated_pages),
 228             (wikiredircat, generated_redirects)]:
 229         catpages = site.Pages['Category:' + tcat]
 230         existingpages = []
 231         for page in catpages:
 232             existingpages.append(page.name)
 233             if page.name in genp:
 234                 pagetxt = page.edit()
 235                 if pagetxt != genp[page.name]:
 236                     print "Updating modified page " + page.name
 237                     page.save(genp[page.name], summary='Auto-updated')
 238                 else:
 239                     if options.verbose:
 240                         print "Page " + page.name + " is unchanged"
 241             else:
 242                 print "Deleting page " + page.name
 243                 page.delete('No longer published')
 244         for pagename, text in genp.items():
 245             if options.verbose:
 246                 print "Checking " + pagename
 247             if not pagename in existingpages:
 248                 print "Creating page " + pagename
 249                 try:
 250                     newpage = site.Pages[pagename]
 251                     newpage.save(text, summary='Auto-created')
 252                 except:
 253                     print "...FAILED to create page"
 254
 255     # Purge server cache to ensure counts are up to date
 256     site.pages['Repository Maintenance'].purge()
 257
 258 def delete_disabled_builds(apps, apkcache, repodirs):
 259     """Delete disabled build outputs.
 260
 261     :param apps: list of all applications, as per metadata.read_metadata
 262     :param apkcache: current apk cache information
 263     :param repodirs: the repo directories to process
 264     """
 265     for app in apps:
 266         for build in app['builds']:
 267             if 'disable' in build:
 268                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 269                 for repodir in repodirs:
 270                     apkpath = os.path.join(repodir, apkfilename)
 271                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 272                     for name in [apkpath, srcpath]:
 273                         if os.path.exists(name):
 274                             print "Deleting disabled build output " + apkfilename
 275                             os.remove(name)
 276                 if apkfilename in apkcache:
 277                     del apkcache[apkfilename]
 278
 279 def resize_icon(iconpath, density):
 280
 281     if not os.path.isfile(iconpath):
 282         return
 283
 284     try:
 285         im = Image.open(iconpath)
 286         size = dpi_to_px(density)
 287
 288         if any(length > size for length in im.size):
 289             oldsize = im.size
 290             im.thumbnail((size, size), Image.ANTIALIAS)
 291             print iconpath, "was too large at", oldsize, "- new size is", im.size
 292             im.save(iconpath, "PNG")
 293
 294         else:
 295             if options.verbose:
 296                 print iconpath, "is small enough:", im.size
 297
 298     except Exception,e:
 299         print "WARNING: Failed resizing {0} - {1}".format(iconpath, e)
 300
 301 def resize_all_icons(repodirs):
 302     """Resize all icons that exceed the max size
 303
 304     :param repodirs: the repo directories to process
 305     """
 306     for repodir in repodirs:
 307         for density in get_densities():
 308             icon_dir = get_icon_dir(repodir, density)
 309             icon_glob = os.path.join(icon_dir, '*.png')
 310             for iconpath in glob.glob(icon_glob):
 311                 resize_icon(iconpath, density)
 312
 313 def scan_apks(apps, apkcache, repodir, knownapks):
 314     """Scan the apks in the given repo directory.
 315
 316     This also extracts the icons.
 317
 318     :param apps: list of all applications, as per metadata.read_metadata
 319     :param apkcache: current apk cache information
 320     :param repodir: repo directory to scan
 321     :param knownapks: known apks info
 322     :returns: (apks, cachechanged) where apks is a list of apk information,
 323               and cachechanged is True if the apkcache got changed.
 324     """
 325
 326     cachechanged = False
 327
 328     icon_dirs = get_icon_dirs(repodir)
 329     for icon_dir in icon_dirs:
 330         if os.path.exists(icon_dir):
 331             if options.clean:
 332                 shutil.rmtree(icon_dir)
 333                 os.makedirs(icon_dir)
 334         else:
 335             os.makedirs(icon_dir)
 336
 337     apks = []
 338     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 339     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 340     vername_pat = re.compile(".*versionName='([^']*)'.*")
 341     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 342     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 343     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 344     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 345     string_pat = re.compile(".*'([^']*)'.*")
 346     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 347
 348         apkfilename = apkfile[len(repodir) + 1:]
 349         if ' ' in apkfilename:
 350             print "No spaces in APK filenames!"
 351             sys.exit(1)
 352
 353         if apkfilename in apkcache:
 354             if options.verbose:
 355                 print "Reading " + apkfilename + " from cache"
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             if not options.quiet:
 361                 print "Processing " + apkfilename
 362             thisinfo = {}
 363             thisinfo['apkname'] = apkfilename
 364             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 365             if os.path.exists(os.path.join(repodir, srcfilename)):
 366                 thisinfo['srcname'] = srcfilename
 367             thisinfo['size'] = os.path.getsize(apkfile)
 368             thisinfo['permissions'] = []
 369             thisinfo['features'] = []
 370             thisinfo['icons_src'] = {}
 371             thisinfo['icons'] = {}
 372             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 373                                   'dump', 'badging', apkfile],
 374                                  stdout=subprocess.PIPE)
 375             output = p.communicate()[0]
 376             if p.returncode != 0:
 377                 print "ERROR: Failed to get apk information"
 378                 sys.exit(1)
 379             for line in output.splitlines():
 380                 if line.startswith("package:"):
 381                     try:
 382                         thisinfo['id'] = re.match(name_pat, line).group(1)
 383                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 384                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 385                     except Exception, e:
 386                         print "Package matching failed: " + str(e)
 387                         print "Line was: " + line
 388                         sys.exit(1)
 389                 elif line.startswith("application:"):
 390                     thisinfo['name'] = re.match(label_pat, line).group(1)
 391                     # Keep path to non-dpi icon in case we need it
 392                     match = re.match(icon_pat_nodpi, line)
 393                     if match:
 394                         thisinfo['icons_src']['-1'] = match.group(1)
 395                 elif line.startswith("launchable-activity:"):
 396                     # Only use launchable-activity as fallback to application
 397                     if '-1' not in thisinfo['icons_src']:
 398                         match = re.match(icon_pat_nodpi, line)
 399                         if match:
 400                             thisinfo['icons_src']['-1'] = match.group(1)
 401                 elif line.startswith("application-icon-"):
 402                     match = re.match(icon_pat, line)
 403                     if match:
 404                         density = match.group(1)
 405                         path = match.group(2)
 406                         thisinfo['icons_src'][density] = path
 407                 elif line.startswith("sdkVersion:"):
 408                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 print "  WARNING: no SDK version information found"
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 print "ERROR: getsig.class not found. To fix:"
 450                 print "\tcd " + getsig_dir
 451                 print "\t./make.sh"
 452                 sys.exit(1)
 453             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 454                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 455             output = p.communicate()[0]
 456             if options.verbose:
 457                 print output
 458             if p.returncode != 0 or not output.startswith('Result:'):
 459                 print "ERROR: Failed to get apk signature"
 460                 sys.exit(1)
 461             thisinfo['sig'] = output[7:].strip()
 462
 463             apk = zipfile.ZipFile(apkfile, 'r')
 464
 465             iconfilename = "%s.%s.png" % (
 466                     thisinfo['id'],
 467                     thisinfo['versioncode'])
 468
 469             # Extract the icon file...
 470             densities = get_densities()
 471             empty_densities = []
 472             for density in densities:
 473                 if density not in thisinfo['icons_src']:
 474                     empty_densities.append(density)
 475                     continue
 476                 iconsrc = thisinfo['icons_src'][density]
 477                 icon_dir = get_icon_dir(repodir, density)
 478                 icondest = os.path.join(icon_dir, iconfilename)
 479
 480                 try:
 481                     iconfile = open(icondest, 'wb')
 482                     iconfile.write(apk.read(iconsrc))
 483                     iconfile.close()
 484                     thisinfo['icons'][density] = iconfilename
 485
 486                 except:
 487                     print "WARNING: Error retrieving icon file"
 488                     del thisinfo['icons'][density]
 489                     del thisinfo['icons_src'][density]
 490                     empty_densities.append(density)
 491
 492             if '-1' in thisinfo['icons_src']:
 493                 iconsrc = thisinfo['icons_src']['-1']
 494                 iconpath = os.path.join(
 495                         get_icon_dir(repodir, None), iconfilename)
 496                 iconfile = open(iconpath, 'wb')
 497                 iconfile.write(apk.read(iconsrc))
 498                 iconfile.close()
 499                 try:
 500                     im = Image.open(iconpath)
 501                     dpi = px_to_dpi(im.size[0])
 502                     for density in densities:
 503                         if density in thisinfo['icons']:
 504                             break
 505                         if density == densities[-1] or dpi >= int(density):
 506                             thisinfo['icons'][density] = iconfilename
 507                             shutil.move(iconpath,
 508                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 509                             empty_densities.remove(density)
 510                             break
 511                 except Exception,e:
 512                     print "WARNING: Failed reading {0} - {1}".format(iconpath, e)
 513
 514             if thisinfo['icons']:
 515                 thisinfo['icon'] = iconfilename
 516
 517             apk.close()
 518
 519             # First try resizing down to not lose quality
 520             last_density = None
 521             for density in densities:
 522                 if density not in empty_densities:
 523                     last_density = density
 524                     continue
 525                 if last_density is None:
 526                     continue
 527                 if options.verbose:
 528                     print "Density %s not available, resizing down from %s" % (
 529                             density, last_density)
 530
 531                 last_iconpath = os.path.join(
 532                         get_icon_dir(repodir, last_density), iconfilename)
 533                 iconpath = os.path.join(
 534                         get_icon_dir(repodir, density), iconfilename)
 535                 try:
 536                     im = Image.open(last_iconpath)
 537                 except:
 538                     print "WARNING: Invalid image file at %s" % last_iconpath
 539                     continue
 540
 541                 size = dpi_to_px(density)
 542
 543                 im.thumbnail((size, size), Image.ANTIALIAS)
 544                 im.save(iconpath, "PNG")
 545                 empty_densities.remove(density)
 546
 547             # Then just copy from the highest resolution available
 548             last_density = None
 549             for density in reversed(densities):
 550                 if density not in empty_densities:
 551                     last_density = density
 552                     continue
 553                 if last_density is None:
 554                     continue
 555                 if options.verbose:
 556                     print "Density %s not available, copying from lower density %s" % (
 557                             density, last_density)
 558
 559                 shutil.copyfile(
 560                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 561                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 562
 563                 empty_densities.remove(density)
 564
 565             for density in densities:
 566                 icon_dir = get_icon_dir(repodir, density)
 567                 icondest = os.path.join(icon_dir, iconfilename)
 568                 resize_icon(icondest, density)
 569
 570             # Copy from icons-mdpi to icons since mdpi is the baseline density
 571             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 572             if os.path.isfile(baseline):
 573                 shutil.copyfile(baseline,
 574                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 575
 576             # Record in known apks, getting the added date at the same time..
 577             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 578             if added:
 579                 thisinfo['added'] = added
 580
 581             apkcache[apkfilename] = thisinfo
 582             cachechanged = True
 583
 584         apks.append(thisinfo)
 585
 586     return apks, cachechanged
 587
 588
 589 repo_pubkey_fingerprint = None
 590
 591 def make_index(apps, apks, repodir, archive, categories):
 592     """Make a repo index.
 593
 594     :param apps: fully populated apps list
 595     :param apks: full populated apks list
 596     :param repodir: the repo directory
 597     :param archive: True if this is the archive repo, False if it's the
 598                     main one.
 599     :param categories: list of categories
 600     """
 601
 602     doc = Document()
 603
 604     def addElement(name, value, doc, parent):
 605         el = doc.createElement(name)
 606         el.appendChild(doc.createTextNode(value))
 607         parent.appendChild(el)
 608     def addElementCDATA(name, value, doc, parent):
 609         el = doc.createElement(name)
 610         el.appendChild(doc.createCDATASection(value))
 611         parent.appendChild(el)
 612
 613     root = doc.createElement("fdroid")
 614     doc.appendChild(root)
 615
 616     repoel = doc.createElement("repo")
 617
 618     if archive:
 619         repoel.setAttribute("name", config['archive_name'])
 620         if config['repo_maxage'] != 0:
 621             repoel.setAttribute("maxage", str(config['repo_maxage']))
 622         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 623         repoel.setAttribute("url", config['archive_url'])
 624         addElement('description', config['archive_description'], doc, repoel)
 625
 626     else:
 627         repoel.setAttribute("name", config['repo_name'])
 628         if config['repo_maxage'] != 0:
 629             repoel.setAttribute("maxage", str(config['repo_maxage']))
 630         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 631         repoel.setAttribute("url", config['repo_url'])
 632         addElement('description', config['repo_description'], doc, repoel)
 633
 634     repoel.setAttribute("version", "11")
 635     repoel.setAttribute("timestamp", str(int(time.time())))
 636
 637     if config['repo_keyalias']:
 638
 639         # Generate a certificate fingerprint the same way keytool does it
 640         # (but with slightly different formatting)
 641         def cert_fingerprint(data):
 642             digest = hashlib.sha1(data).digest()
 643             ret = []
 644             for i in range(4):
 645                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 646             return " ".join(ret)
 647
 648         def extract_pubkey():
 649             p = subprocess.Popen(['keytool', '-exportcert',
 650                                   '-alias', config['repo_keyalias'],
 651                                   '-keystore', config['keystore'],
 652                                   '-storepass', config['keystorepass']],
 653                                  stdout=subprocess.PIPE)
 654             cert = p.communicate()[0]
 655             if p.returncode != 0:
 656                 print "ERROR: Failed to get repo pubkey"
 657                 sys.exit(1)
 658             global repo_pubkey_fingerprint
 659             repo_pubkey_fingerprint = cert_fingerprint(cert)
 660             return "".join("%02x" % ord(b) for b in cert)
 661
 662         repoel.setAttribute("pubkey", extract_pubkey())
 663
 664     root.appendChild(repoel)
 665
 666     for app in apps:
 667
 668         if app['Disabled'] is not None:
 669             continue
 670
 671         # Get a list of the apks for this app...
 672         apklist = []
 673         for apk in apks:
 674             if apk['id'] == app['id']:
 675                 apklist.append(apk)
 676
 677         if len(apklist) == 0:
 678             continue
 679
 680         apel = doc.createElement("application")
 681         apel.setAttribute("id", app['id'])
 682         root.appendChild(apel)
 683
 684         addElement('id', app['id'], doc, apel)
 685         if 'added' in app:
 686             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 687         if 'lastupdated' in app:
 688             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 689         addElement('name', app['Name'], doc, apel)
 690         addElement('summary', app['Summary'], doc, apel)
 691         if app['icon']:
 692             addElement('icon', app['icon'], doc, apel)
 693         def linkres(link):
 694             for app in apps:
 695                 if app['id'] == link:
 696                     return ("fdroid.app:" + link, app['Name'])
 697             raise MetaDataException("Cannot resolve app id " + link)
 698         addElement('desc',
 699                 metadata.description_html(app['Description'], linkres), doc, apel)
 700         addElement('license', app['License'], doc, apel)
 701         if 'Categories' in app:
 702             appcategories = [c.strip() for c in app['Categories'].split(',')]
 703             addElement('categories', ','.join(appcategories), doc, apel)
 704             # We put the first (primary) category in LAST, which will have
 705             # the desired effect of making clients that only understand one
 706             # category see that one.
 707             addElement('category', appcategories[0], doc, apel)
 708         addElement('web', app['Web Site'], doc, apel)
 709         addElement('source', app['Source Code'], doc, apel)
 710         addElement('tracker', app['Issue Tracker'], doc, apel)
 711         if app['Donate']:
 712             addElement('donate', app['Donate'], doc, apel)
 713         if app['Bitcoin']:
 714             addElement('bitcoin', app['Bitcoin'], doc, apel)
 715         if app['Litecoin']:
 716             addElement('litecoin', app['Litecoin'], doc, apel)
 717         if app['Dogecoin']:
 718             addElement('dogecoin', app['Dogecoin'], doc, apel)
 719         if app['FlattrID']:
 720             addElement('flattr', app['FlattrID'], doc, apel)
 721
 722         # These elements actually refer to the current version (i.e. which
 723         # one is recommended. They are historically mis-named, and need
 724         # changing, but stay like this for now to support existing clients.
 725         addElement('marketversion', app['Current Version'], doc, apel)
 726         addElement('marketvercode', app['Current Version Code'], doc, apel)
 727
 728         if app['AntiFeatures']:
 729             af = app['AntiFeatures'].split(',')
 730             # TODO: Temporarily not including UpstreamNonFree in the index,
 731             # because current F-Droid clients do not understand it, and also
 732             # look ugly when they encounter an unknown antifeature. This
 733             # filtering can be removed in time...
 734             if 'UpstreamNonFree' in af:
 735                 af.remove('UpstreamNonFree')
 736             if af:
 737                 addElement('antifeatures', ','.join(af), doc, apel)
 738         if app['Provides']:
 739             pv = app['Provides'].split(',')
 740             addElement('provides', ','.join(pv), doc, apel)
 741         if app['Requires Root']:
 742             addElement('requirements', 'root', doc, apel)
 743
 744         # Sort the apk list into version order, just so the web site
 745         # doesn't have to do any work by default...
 746         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 747
 748         # Check for duplicates - they will make the client unhappy...
 749         for i in range(len(apklist) - 1):
 750             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 751                 print "ERROR - duplicate versions"
 752                 print apklist[i]['apkname']
 753                 print apklist[i+1]['apkname']
 754                 sys.exit(1)
 755
 756         for apk in apklist:
 757             apkel = doc.createElement("package")
 758             apel.appendChild(apkel)
 759             addElement('version', apk['version'], doc, apkel)
 760             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 761             addElement('apkname', apk['apkname'], doc, apkel)
 762             if 'srcname' in apk:
 763                 addElement('srcname', apk['srcname'], doc, apkel)
 764             for hash_type in ['sha256']:
 765                 if not hash_type in apk:
 766                     continue
 767                 hashel = doc.createElement("hash")
 768                 hashel.setAttribute("type", hash_type)
 769                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 770                 apkel.appendChild(hashel)
 771             addElement('sig', apk['sig'], doc, apkel)
 772             addElement('size', str(apk['size']), doc, apkel)
 773             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 774             if 'added' in apk:
 775                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 776             if app['Requires Root']:
 777                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 778                     apk['permissions'].append('ACCESS_SUPERUSER')
 779
 780             if len(apk['permissions']) > 0:
 781                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 782             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 783                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 784             if len(apk['features']) > 0:
 785                 addElement('features', ','.join(apk['features']), doc, apkel)
 786
 787     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 788     if options.pretty:
 789         output = doc.toprettyxml()
 790     else:
 791         output = doc.toxml()
 792     of.write(output)
 793     of.close()
 794
 795     if config['repo_keyalias'] is not None:
 796
 797         if not options.quiet:
 798             print "Creating signed index."
 799             print "Key fingerprint:", repo_pubkey_fingerprint
 800
 801         #Create a jar of the index...
 802         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 803             cwd=repodir, stdout=subprocess.PIPE)
 804         output = p.communicate()[0]
 805         if options.verbose:
 806             print output
 807         if p.returncode != 0:
 808             print "ERROR: Failed to create jar file"
 809             sys.exit(1)
 810
 811         # Sign the index...
 812         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 813             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 814             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 815             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 816         output = p.communicate()[0]
 817         if p.returncode != 0:
 818             print "Failed to sign index"
 819             print output
 820             sys.exit(1)
 821         if options.verbose:
 822             print output
 823
 824     # Copy the repo icon into the repo directory...
 825     icon_dir = os.path.join(repodir ,'icons')
 826     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 827     shutil.copyfile(config['repo_icon'], iconfilename)
 828
 829     # Write a category list in the repo to allow quick access...
 830     catdata = ''
 831     for cat in categories:
 832         catdata += cat + '\n'
 833     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 834     f.write(catdata)
 835     f.close()
 836
 837
 838
 839 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 840
 841     for app in apps:
 842
 843         # Get a list of the apks for this app...
 844         apklist = []
 845         for apk in apks:
 846             if apk['id'] == app['id']:
 847                 apklist.append(apk)
 848
 849         # Sort the apk list into version order...
 850         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 851
 852         if app['Archive Policy']:
 853             keepversions = int(app['Archive Policy'][:-9])
 854         else:
 855             keepversions = defaultkeepversions
 856
 857         if len(apklist) > keepversions:
 858             for apk in apklist[keepversions:]:
 859                 print "Moving " + apk['apkname'] + " to archive"
 860                 shutil.move(os.path.join(repodir, apk['apkname']),
 861                     os.path.join(archivedir, apk['apkname']))
 862                 if 'srcname' in apk:
 863                     shutil.move(os.path.join(repodir, apk['srcname']),
 864                         os.path.join(archivedir, apk['srcname']))
 865                 archapks.append(apk)
 866                 apks.remove(apk)
 867
 868
 869 config = None
 870 options = None
 871
 872 def main():
 873
 874     global config, options
 875
 876     # Parse command line...
 877     parser = OptionParser()
 878     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 879                       help="Create skeleton metadata files that are missing")
 880     parser.add_option("-v", "--verbose", action="store_true", default=False,
 881                       help="Spew out even more information than normal")
 882     parser.add_option("-q", "--quiet", action="store_true", default=False,
 883                       help="No output, except for warnings and errors")
 884     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 885                       help="Report on build data status")
 886     parser.add_option("-i", "--interactive", default=False, action="store_true",
 887                       help="Interactively ask about things that need updating.")
 888     parser.add_option("-I", "--icons", action="store_true", default=False,
 889                       help="Resize all the icons exceeding the max pixel size and exit")
 890     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 891                       help="Specify editor to use in interactive mode. Default "+
 892                           "is /etc/alternatives/editor")
 893     parser.add_option("-w", "--wiki", default=False, action="store_true",
 894                       help="Update the wiki")
 895     parser.add_option("", "--pretty", action="store_true", default=False,
 896                       help="Produce human-readable index.xml")
 897     parser.add_option("--clean", action="store_true", default=False,
 898                       help="Clean update - don't uses caches, reprocess all apks")
 899     (options, args) = parser.parse_args()
 900
 901     config = common.read_config(options)
 902
 903     repodirs = ['repo']
 904     if config['archive_older'] != 0:
 905         repodirs.append('archive')
 906         if not os.path.exists('archive'):
 907             os.mkdir('archive')
 908
 909     if options.icons:
 910         resize_all_icons(repodirs)
 911         sys.exit(0)
 912
 913     # Get all apps...
 914     apps = metadata.read_metadata()
 915
 916     # Generate a list of categories...
 917     categories = []
 918     for app in apps:
 919         cats = app['Categories'].split(',')
 920         for cat in cats:
 921             if cat not in categories:
 922                 categories.append(cat)
 923
 924     # Read known apks data (will be updated and written back when we've finished)
 925     knownapks = common.KnownApks()
 926
 927     # Gather information about all the apk files in the repo directory, using
 928     # cached data if possible.
 929     apkcachefile = os.path.join('tmp', 'apkcache')
 930     if not options.clean and os.path.exists(apkcachefile):
 931         with open(apkcachefile, 'rb') as cf:
 932             apkcache = pickle.load(cf)
 933     else:
 934         apkcache = {}
 935     cachechanged = False
 936
 937     delete_disabled_builds(apps, apkcache, repodirs)
 938
 939     # Scan all apks in the main repo
 940     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 941     if cc:
 942         cachechanged = True
 943
 944     # Scan the archive repo for apks as well
 945     if len(repodirs) > 1:
 946         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 947         if cc:
 948             cachechanged = True
 949     else:
 950         archapks = []
 951
 952     # Some information from the apks needs to be applied up to the application
 953     # level. When doing this, we use the info from the most recent version's apk.
 954     # We deal with figuring out when the app was added and last updated at the
 955     # same time.
 956     for app in apps:
 957         bestver = 0
 958         added = None
 959         lastupdated = None
 960         for apk in apks + archapks:
 961             if apk['id'] == app['id']:
 962                 if apk['versioncode'] > bestver:
 963                     bestver = apk['versioncode']
 964                     bestapk = apk
 965
 966                 if 'added' in apk:
 967                     if not added or apk['added'] < added:
 968                         added = apk['added']
 969                     if not lastupdated or apk['added'] > lastupdated:
 970                         lastupdated = apk['added']
 971
 972         if added:
 973             app['added'] = added
 974         else:
 975             if options.verbose:
 976                 print "WARNING: Don't know when " + app['id'] + " was added"
 977         if lastupdated:
 978             app['lastupdated'] = lastupdated
 979         else:
 980             if options.verbose:
 981                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 982
 983         if bestver == 0:
 984             if app['Name'] is None:
 985                 app['Name'] = app['id']
 986             app['icon'] = None
 987             if options.verbose and app['Disabled'] is None:
 988                 print "WARNING: Application " + app['id'] + " has no packages"
 989         else:
 990             if app['Name'] is None:
 991                 app['Name'] = bestapk['name']
 992             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 993
 994     # Sort the app list by name, then the web site doesn't have to by default.
 995     # (we had to wait until we'd scanned the apks to do this, because mostly the
 996     # name comes from there!)
 997     apps = sorted(apps, key=lambda app: app['Name'].upper())
 998
 999     # Generate warnings for apk's with no metadata (or create skeleton
1000     # metadata files, if requested on the command line)
1001     for apk in apks:
1002         found = False
1003         for app in apps:
1004             if app['id'] == apk['id']:
1005                 found = True
1006                 break
1007         if not found:
1008             if options.createmeta:
1009                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1010                 f.write("License:Unknown\n")
1011                 f.write("Web Site:\n")
1012                 f.write("Source Code:\n")
1013                 f.write("Issue Tracker:\n")
1014                 f.write("Summary:" + apk['name'] + "\n")
1015                 f.write("Description:\n")
1016                 f.write(apk['name'] + "\n")
1017                 f.write(".\n")
1018                 f.close()
1019                 print "Generated skeleton metadata for " + apk['id']
1020             else:
1021                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
1022                 print "       " + apk['name'] + " - " + apk['version']
1023
1024     if len(repodirs) > 1:
1025         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1026
1027     # Make the index for the main repo...
1028     make_index(apps, apks, repodirs[0], False, categories)
1029
1030     # If there's an archive repo,  make the index for it. We already scanned it
1031     # earlier on.
1032     if len(repodirs) > 1:
1033         make_index(apps, archapks, repodirs[1], True, categories)
1034
1035     if config['update_stats']:
1036
1037         # Update known apks info...
1038         knownapks.writeifchanged()
1039
1040         # Generate latest apps data for widget
1041         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1042             data = ''
1043             for line in file(os.path.join('stats', 'latestapps.txt')):
1044                 appid = line.rstrip()
1045                 data += appid + "\t"
1046                 for app in apps:
1047                     if app['id'] == appid:
1048                         data += app['Name'] + "\t"
1049                         if app['icon'] is not None:
1050                             data += app['icon'] + "\t"
1051                         data += app['License'] + "\n"
1052                         break
1053             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1054             f.write(data)
1055             f.close()
1056
1057     if cachechanged:
1058         with open(apkcachefile, 'wb') as cf:
1059             pickle.dump(apkcache, cf)
1060
1061     # Update the wiki...
1062     if options.wiki:
1063         update_wiki(apps, apks + archapks)
1064
1065     print "Finished."
1066
1067 if __name__ == "__main__":
1068     main()
1069