fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def dpi_to_px(density):
  42     return (int(density) * 48) / 160
  43
  44 def px_to_dpi(px):
  45     return (int(px) * 160) / 48
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         elif validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         elif cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         elif buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 191             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 192
 193         # We can't have underscores in the page name, even if they're in
 194         # the package ID, because MediaWiki messes with them...
 195         pagename = app['id'].replace('_', ' ')
 196
 197         # Drop a trailing newline, because mediawiki is going to drop it anyway
 198         # and it we don't we'll think the page has changed when it hasn't...
 199         if wikidata.endswith('\n'):
 200             wikidata = wikidata[:-1]
 201
 202         generated_pages[pagename] = wikidata
 203
 204         # Make a redirect from the name to the ID too, unless there's
 205         # already an existing page with the name and it isn't a redirect.
 206         noclobber = False
 207         apppagename = app['Name'].replace('_', ' ')
 208         apppagename = apppagename.replace('{', '')
 209         apppagename = apppagename.replace('}', ' ')
 210         apppagename = apppagename.replace(':', ' ')
 211         # Drop double spaces caused mostly by replacing ':' above
 212         apppagename = apppagename.replace('  ', ' ')
 213         for expagename in site.allpages(prefix=apppagename,
 214                 filterredir='nonredirects', generator=False):
 215             if expagename == apppagename:
 216                 noclobber = True
 217         # Another reason not to make the redirect page is if the app name
 218         # is the same as it's ID, because that will overwrite the real page
 219         # with an redirect to itself! (Although it seems like an odd
 220         # scenario this happens a lot, e.g. where there is metadata but no
 221         # builds or binaries to extract a name from.
 222         if apppagename == pagename:
 223             noclobber = True
 224         if not noclobber:
 225             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 226
 227     for tcat, genp in [(wikicat, generated_pages),
 228             (wikiredircat, generated_redirects)]:
 229         catpages = site.Pages['Category:' + tcat]
 230         existingpages = []
 231         for page in catpages:
 232             existingpages.append(page.name)
 233             if page.name in genp:
 234                 pagetxt = page.edit()
 235                 if pagetxt != genp[page.name]:
 236                     print "Updating modified page " + page.name
 237                     page.save(genp[page.name], summary='Auto-updated')
 238                 else:
 239                     if options.verbose:
 240                         print "Page " + page.name + " is unchanged"
 241             else:
 242                 print "Deleting page " + page.name
 243                 page.delete('No longer published')
 244         for pagename, text in genp.items():
 245             if options.verbose:
 246                 print "Checking " + pagename
 247             if not pagename in existingpages:
 248                 print "Creating page " + pagename
 249                 try:
 250                     newpage = site.Pages[pagename]
 251                     newpage.save(text, summary='Auto-created')
 252                 except:
 253                     print "...FAILED to create page"
 254
 255     # Purge server cache to ensure counts are up to date
 256     site.pages['Repository Maintenance'].purge()
 257
 258 def delete_disabled_builds(apps, apkcache, repodirs):
 259     """Delete disabled build outputs.
 260
 261     :param apps: list of all applications, as per metadata.read_metadata
 262     :param apkcache: current apk cache information
 263     :param repodirs: the repo directories to process
 264     """
 265     for app in apps:
 266         for build in app['builds']:
 267             if 'disable' in build:
 268                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 269                 for repodir in repodirs:
 270                     apkpath = os.path.join(repodir, apkfilename)
 271                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 272                     for name in [apkpath, srcpath]:
 273                         if os.path.exists(name):
 274                             print "Deleting disabled build output " + apkfilename
 275                             os.remove(name)
 276                 if apkfilename in apkcache:
 277                     del apkcache[apkfilename]
 278
 279 def resize_icon(iconpath, density):
 280
 281     if not os.path.isfile(iconpath):
 282         return
 283
 284     try:
 285         im = Image.open(iconpath)
 286         size = dpi_to_px(density)
 287
 288         if any(length > size for length in im.size):
 289             oldsize = im.size
 290             im.thumbnail((size, size), Image.ANTIALIAS)
 291             print iconpath, "was too large at", oldsize, "- new size is", im.size
 292             im.save(iconpath, "PNG")
 293
 294         else:
 295             if options.verbose:
 296                 print iconpath, "is small enough:", im.size
 297
 298     except Exception,e:
 299         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 300
 301 def resize_all_icons(repodirs):
 302     """Resize all icons that exceed the max size
 303
 304     :param repodirs: the repo directories to process
 305     """
 306     for repodir in repodirs:
 307         for density in get_densities():
 308             icon_dir = get_icon_dir(repodir, density)
 309             icon_glob = os.path.join(icon_dir, '*.png')
 310             for iconpath in glob.glob(icon_glob):
 311                 resize_icon(iconpath, density)
 312
 313 def scan_apks(apps, apkcache, repodir, knownapks):
 314     """Scan the apks in the given repo directory.
 315
 316     This also extracts the icons.
 317
 318     :param apps: list of all applications, as per metadata.read_metadata
 319     :param apkcache: current apk cache information
 320     :param repodir: repo directory to scan
 321     :param knownapks: known apks info
 322     :returns: (apks, cachechanged) where apks is a list of apk information,
 323               and cachechanged is True if the apkcache got changed.
 324     """
 325
 326     cachechanged = False
 327
 328     icon_dirs = get_icon_dirs(repodir)
 329     for icon_dir in icon_dirs:
 330         if os.path.exists(icon_dir):
 331             if options.clean:
 332                 shutil.rmtree(icon_dir)
 333                 os.makedirs(icon_dir)
 334         else:
 335             os.makedirs(icon_dir)
 336
 337     apks = []
 338     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 339     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 340     vername_pat = re.compile(".*versionName='([^']*)'.*")
 341     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 342     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 343     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 344     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 345     string_pat = re.compile(".*'([^']*)'.*")
 346     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 347
 348         apkfilename = apkfile[len(repodir) + 1:]
 349         if ' ' in apkfilename:
 350             print "No spaces in APK filenames!"
 351             sys.exit(1)
 352
 353         if apkfilename in apkcache:
 354             if options.verbose:
 355                 print "Reading " + apkfilename + " from cache"
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             if not options.quiet:
 361                 print "Processing " + apkfilename
 362             thisinfo = {}
 363             thisinfo['apkname'] = apkfilename
 364             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 365             if os.path.exists(os.path.join(repodir, srcfilename)):
 366                 thisinfo['srcname'] = srcfilename
 367             thisinfo['size'] = os.path.getsize(apkfile)
 368             thisinfo['permissions'] = []
 369             thisinfo['features'] = []
 370             thisinfo['icons_src'] = {}
 371             thisinfo['icons'] = {}
 372             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 373                                   'dump', 'badging', apkfile],
 374                                  stdout=subprocess.PIPE)
 375             output = p.communicate()[0]
 376             if p.returncode != 0:
 377                 print "ERROR: Failed to get apk information"
 378                 sys.exit(1)
 379             for line in output.splitlines():
 380                 if line.startswith("package:"):
 381                     try:
 382                         thisinfo['id'] = re.match(name_pat, line).group(1)
 383                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 384                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 385                     except Exception, e:
 386                         print "Package matching failed: " + str(e)
 387                         print "Line was: " + line
 388                         sys.exit(1)
 389                 elif line.startswith("application:"):
 390                     thisinfo['name'] = re.match(label_pat, line).group(1)
 391                     # Keep path to non-dpi icon in case we need it
 392                     match = re.match(icon_pat_nodpi, line)
 393                     if match:
 394                         thisinfo['icons_src']['-1'] = match.group(1)
 395                 elif line.startswith("launchable-activity:"):
 396                     # Only use launchable-activity as fallback to application
 397                     if '-1' not in thisinfo['icons_src']:
 398                         match = re.match(icon_pat_nodpi, line)
 399                         if match:
 400                             thisinfo['icons_src']['-1'] = match.group(1)
 401                 elif line.startswith("application-icon-"):
 402                     match = re.match(icon_pat, line)
 403                     if match:
 404                         density = match.group(1)
 405                         path = match.group(2)
 406                         thisinfo['icons_src'][density] = path
 407                 elif line.startswith("sdkVersion:"):
 408                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 print "  WARNING: no SDK version information found"
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 print "ERROR: getsig.class not found. To fix:"
 450                 print "\tcd " + getsig_dir
 451                 print "\t./make.sh"
 452                 sys.exit(1)
 453             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 454                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 455             output = p.communicate()[0]
 456             if options.verbose:
 457                 print output
 458             if p.returncode != 0 or not output.startswith('Result:'):
 459                 print "ERROR: Failed to get apk signature"
 460                 sys.exit(1)
 461             thisinfo['sig'] = output[7:].strip()
 462
 463             apk = zipfile.ZipFile(apkfile, 'r')
 464
 465             iconfilename = "%s.%s.png" % (
 466                     thisinfo['id'],
 467                     thisinfo['versioncode'])
 468
 469             # Extract the icon file...
 470             densities = get_densities()
 471             empty_densities = []
 472             for density in densities:
 473                 if density not in thisinfo['icons_src']:
 474                     empty_densities.append(density)
 475                     continue
 476                 iconsrc = thisinfo['icons_src'][density]
 477                 icon_dir = get_icon_dir(repodir, density)
 478                 icondest = os.path.join(icon_dir, iconfilename)
 479
 480                 try:
 481                     iconfile = open(icondest, 'wb')
 482                     iconfile.write(apk.read(iconsrc))
 483                     iconfile.close()
 484                     thisinfo['icons'][density] = iconfilename
 485
 486                 except:
 487                     print "WARNING: Error retrieving icon file"
 488                     del thisinfo['icons'][density]
 489                     del thisinfo['icons_src'][density]
 490                     empty_densities.append(density)
 491
 492             if '-1' in thisinfo['icons_src']:
 493                 iconsrc = thisinfo['icons_src']['-1']
 494                 iconpath = os.path.join(
 495                         get_icon_dir(repodir, None), iconfilename)
 496                 iconfile = open(iconpath, 'wb')
 497                 iconfile.write(apk.read(iconsrc))
 498                 iconfile.close()
 499                 im = Image.open(iconpath)
 500                 dpi = px_to_dpi(im.size[0])
 501                 for density in densities:
 502                     if density in thisinfo['icons']:
 503                         break
 504                     if density == densities[-1] or dpi >= int(density):
 505                         thisinfo['icons'][density] = iconfilename
 506                         shutil.move(iconpath,
 507                                 os.path.join(get_icon_dir(repodir, density), iconfilename))
 508                         empty_densities.remove(density)
 509                         break
 510
 511             if thisinfo['icons']:
 512                 thisinfo['icon'] = iconfilename
 513
 514             apk.close()
 515
 516             # First try resizing down to not lose quality
 517             last_density = None
 518             for density in densities:
 519                 if density not in empty_densities:
 520                     last_density = density
 521                     continue
 522                 if last_density is None:
 523                     continue
 524                 if options.verbose:
 525                     print "Density %s not available, resizing down from %s" % (
 526                             density, last_density)
 527
 528                 last_iconpath = os.path.join(
 529                         get_icon_dir(repodir, last_density), iconfilename)
 530                 iconpath = os.path.join(
 531                         get_icon_dir(repodir, density), iconfilename)
 532                 im = Image.open(last_iconpath)
 533                 size = dpi_to_px(density)
 534
 535                 im.thumbnail((size, size), Image.ANTIALIAS)
 536                 im.save(iconpath, "PNG")
 537                 empty_densities.remove(density)
 538
 539             # Then just copy from the highest resolution available
 540             last_density = None
 541             for density in reversed(densities):
 542                 if density not in empty_densities:
 543                     last_density = density
 544                     continue
 545                 if last_density is None:
 546                     continue
 547                 if options.verbose:
 548                     print "Density %s not available, copying from lower density %s" % (
 549                             density, last_density)
 550
 551                 shutil.copyfile(
 552                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 553                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 554
 555                 empty_densities.remove(density)
 556
 557             for density in densities:
 558                 icon_dir = get_icon_dir(repodir, density)
 559                 icondest = os.path.join(icon_dir, iconfilename)
 560                 resize_icon(icondest, density)
 561
 562             # Copy from icons-mdpi to icons since mdpi is the baseline density
 563             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 564             if os.path.isfile(baseline):
 565                 shutil.copyfile(baseline,
 566                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 567
 568             # Record in known apks, getting the added date at the same time..
 569             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 570             if added:
 571                 thisinfo['added'] = added
 572
 573             apkcache[apkfilename] = thisinfo
 574             cachechanged = True
 575
 576         apks.append(thisinfo)
 577
 578     return apks, cachechanged
 579
 580
 581 repo_pubkey_fingerprint = None
 582
 583 def make_index(apps, apks, repodir, archive, categories):
 584     """Make a repo index.
 585
 586     :param apps: fully populated apps list
 587     :param apks: full populated apks list
 588     :param repodir: the repo directory
 589     :param archive: True if this is the archive repo, False if it's the
 590                     main one.
 591     :param categories: list of categories
 592     """
 593
 594     doc = Document()
 595
 596     def addElement(name, value, doc, parent):
 597         el = doc.createElement(name)
 598         el.appendChild(doc.createTextNode(value))
 599         parent.appendChild(el)
 600     def addElementCDATA(name, value, doc, parent):
 601         el = doc.createElement(name)
 602         el.appendChild(doc.createCDATASection(value))
 603         parent.appendChild(el)
 604
 605     root = doc.createElement("fdroid")
 606     doc.appendChild(root)
 607
 608     repoel = doc.createElement("repo")
 609
 610     if archive:
 611         repoel.setAttribute("name", config['archive_name'])
 612         if config['repo_maxage'] != 0:
 613             repoel.setAttribute("maxage", str(config['repo_maxage']))
 614         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 615         repoel.setAttribute("url", config['archive_url'])
 616         addElement('description', config['archive_description'], doc, repoel)
 617
 618     else:
 619         repoel.setAttribute("name", config['repo_name'])
 620         if config['repo_maxage'] != 0:
 621             repoel.setAttribute("maxage", str(config['repo_maxage']))
 622         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 623         repoel.setAttribute("url", config['repo_url'])
 624         addElement('description', config['repo_description'], doc, repoel)
 625
 626     repoel.setAttribute("version", "11")
 627     repoel.setAttribute("timestamp", str(int(time.time())))
 628
 629     if config['repo_keyalias']:
 630
 631         # Generate a certificate fingerprint the same way keytool does it
 632         # (but with slightly different formatting)
 633         def cert_fingerprint(data):
 634             digest = hashlib.sha1(data).digest()
 635             ret = []
 636             for i in range(4):
 637                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 638             return " ".join(ret)
 639
 640         def extract_pubkey():
 641             p = subprocess.Popen(['keytool', '-exportcert',
 642                                   '-alias', config['repo_keyalias'],
 643                                   '-keystore', config['keystore'],
 644                                   '-storepass', config['keystorepass']],
 645                                  stdout=subprocess.PIPE)
 646             cert = p.communicate()[0]
 647             if p.returncode != 0:
 648                 print "ERROR: Failed to get repo pubkey"
 649                 sys.exit(1)
 650             global repo_pubkey_fingerprint
 651             repo_pubkey_fingerprint = cert_fingerprint(cert)
 652             return "".join("%02x" % ord(b) for b in cert)
 653
 654         repoel.setAttribute("pubkey", extract_pubkey())
 655
 656     root.appendChild(repoel)
 657
 658     for app in apps:
 659
 660         if app['Disabled'] is not None:
 661             continue
 662
 663         # Get a list of the apks for this app...
 664         apklist = []
 665         for apk in apks:
 666             if apk['id'] == app['id']:
 667                 apklist.append(apk)
 668
 669         if len(apklist) == 0:
 670             continue
 671
 672         apel = doc.createElement("application")
 673         apel.setAttribute("id", app['id'])
 674         root.appendChild(apel)
 675
 676         addElement('id', app['id'], doc, apel)
 677         if 'added' in app:
 678             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 679         if 'lastupdated' in app:
 680             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 681         addElement('name', app['Name'], doc, apel)
 682         addElement('summary', app['Summary'], doc, apel)
 683         if app['icon']:
 684             addElement('icon', app['icon'], doc, apel)
 685         def linkres(link):
 686             for app in apps:
 687                 if app['id'] == link:
 688                     return ("fdroid.app:" + link, app['Name'])
 689             raise MetaDataException("Cannot resolve app id " + link)
 690         addElement('desc',
 691                 metadata.description_html(app['Description'], linkres), doc, apel)
 692         addElement('license', app['License'], doc, apel)
 693         if 'Categories' in app:
 694             appcategories = [c.strip() for c in app['Categories'].split(',')]
 695             addElement('categories', ','.join(appcategories), doc, apel)
 696             # We put the first (primary) category in LAST, which will have
 697             # the desired effect of making clients that only understand one
 698             # category see that one.
 699             addElement('category', appcategories[0], doc, apel)
 700         addElement('web', app['Web Site'], doc, apel)
 701         addElement('source', app['Source Code'], doc, apel)
 702         addElement('tracker', app['Issue Tracker'], doc, apel)
 703         if app['Donate']:
 704             addElement('donate', app['Donate'], doc, apel)
 705         if app['Bitcoin']:
 706             addElement('bitcoin', app['Bitcoin'], doc, apel)
 707         if app['Litecoin']:
 708             addElement('litecoin', app['Litecoin'], doc, apel)
 709         if app['Dogecoin']:
 710             addElement('dogecoin', app['Dogecoin'], doc, apel)
 711         if app['FlattrID']:
 712             addElement('flattr', app['FlattrID'], doc, apel)
 713
 714         # These elements actually refer to the current version (i.e. which
 715         # one is recommended. They are historically mis-named, and need
 716         # changing, but stay like this for now to support existing clients.
 717         addElement('marketversion', app['Current Version'], doc, apel)
 718         addElement('marketvercode', app['Current Version Code'], doc, apel)
 719
 720         if app['AntiFeatures']:
 721             af = app['AntiFeatures'].split(',')
 722             # TODO: Temporarily not including UpstreamNonFree in the index,
 723             # because current F-Droid clients do not understand it, and also
 724             # look ugly when they encounter an unknown antifeature. This
 725             # filtering can be removed in time...
 726             if 'UpstreamNonFree' in af:
 727                 af.remove('UpstreamNonFree')
 728             if af:
 729                 addElement('antifeatures', ','.join(af), doc, apel)
 730         if app['Provides']:
 731             pv = app['Provides'].split(',')
 732             addElement('provides', ','.join(pv), doc, apel)
 733         if app['Requires Root']:
 734             addElement('requirements', 'root', doc, apel)
 735
 736         # Sort the apk list into version order, just so the web site
 737         # doesn't have to do any work by default...
 738         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 739
 740         # Check for duplicates - they will make the client unhappy...
 741         for i in range(len(apklist) - 1):
 742             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 743                 print "ERROR - duplicate versions"
 744                 print apklist[i]['apkname']
 745                 print apklist[i+1]['apkname']
 746                 sys.exit(1)
 747
 748         for apk in apklist:
 749             apkel = doc.createElement("package")
 750             apel.appendChild(apkel)
 751             addElement('version', apk['version'], doc, apkel)
 752             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 753             addElement('apkname', apk['apkname'], doc, apkel)
 754             if 'srcname' in apk:
 755                 addElement('srcname', apk['srcname'], doc, apkel)
 756             for hash_type in ['sha256']:
 757                 if not hash_type in apk:
 758                     continue
 759                 hashel = doc.createElement("hash")
 760                 hashel.setAttribute("type", hash_type)
 761                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 762                 apkel.appendChild(hashel)
 763             addElement('sig', apk['sig'], doc, apkel)
 764             addElement('size', str(apk['size']), doc, apkel)
 765             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 766             if 'added' in apk:
 767                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 768             if app['Requires Root']:
 769                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 770                     apk['permissions'].append('ACCESS_SUPERUSER')
 771
 772             if len(apk['permissions']) > 0:
 773                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 774             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 775                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 776             if len(apk['features']) > 0:
 777                 addElement('features', ','.join(apk['features']), doc, apkel)
 778
 779     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 780     if options.pretty:
 781         output = doc.toprettyxml()
 782     else:
 783         output = doc.toxml()
 784     of.write(output)
 785     of.close()
 786
 787     if config['repo_keyalias'] is not None:
 788
 789         if not options.quiet:
 790             print "Creating signed index."
 791             print "Key fingerprint:", repo_pubkey_fingerprint
 792
 793         #Create a jar of the index...
 794         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 795             cwd=repodir, stdout=subprocess.PIPE)
 796         output = p.communicate()[0]
 797         if options.verbose:
 798             print output
 799         if p.returncode != 0:
 800             print "ERROR: Failed to create jar file"
 801             sys.exit(1)
 802
 803         # Sign the index...
 804         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 805             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 806             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 807             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 808         output = p.communicate()[0]
 809         if p.returncode != 0:
 810             print "Failed to sign index"
 811             print output
 812             sys.exit(1)
 813         if options.verbose:
 814             print output
 815
 816     # Copy the repo icon into the repo directory...
 817     icon_dir = os.path.join(repodir ,'icons')
 818     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 819     shutil.copyfile(config['repo_icon'], iconfilename)
 820
 821     # Write a category list in the repo to allow quick access...
 822     catdata = ''
 823     for cat in categories:
 824         catdata += cat + '\n'
 825     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 826     f.write(catdata)
 827     f.close()
 828
 829
 830
 831 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 832
 833     for app in apps:
 834
 835         # Get a list of the apks for this app...
 836         apklist = []
 837         for apk in apks:
 838             if apk['id'] == app['id']:
 839                 apklist.append(apk)
 840
 841         # Sort the apk list into version order...
 842         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 843
 844         if app['Archive Policy']:
 845             keepversions = int(app['Archive Policy'][:-9])
 846         else:
 847             keepversions = defaultkeepversions
 848
 849         if len(apklist) > keepversions:
 850             for apk in apklist[keepversions:]:
 851                 print "Moving " + apk['apkname'] + " to archive"
 852                 shutil.move(os.path.join(repodir, apk['apkname']),
 853                     os.path.join(archivedir, apk['apkname']))
 854                 if 'srcname' in apk:
 855                     shutil.move(os.path.join(repodir, apk['srcname']),
 856                         os.path.join(archivedir, apk['srcname']))
 857                 archapks.append(apk)
 858                 apks.remove(apk)
 859
 860
 861 config = None
 862 options = None
 863
 864 def main():
 865
 866     global config, options
 867
 868     # Parse command line...
 869     parser = OptionParser()
 870     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 871                       help="Create skeleton metadata files that are missing")
 872     parser.add_option("-v", "--verbose", action="store_true", default=False,
 873                       help="Spew out even more information than normal")
 874     parser.add_option("-q", "--quiet", action="store_true", default=False,
 875                       help="No output, except for warnings and errors")
 876     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 877                       help="Report on build data status")
 878     parser.add_option("-i", "--interactive", default=False, action="store_true",
 879                       help="Interactively ask about things that need updating.")
 880     parser.add_option("-I", "--icons", action="store_true", default=False,
 881                       help="Resize all the icons exceeding the max pixel size and exit")
 882     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 883                       help="Specify editor to use in interactive mode. Default "+
 884                           "is /etc/alternatives/editor")
 885     parser.add_option("-w", "--wiki", default=False, action="store_true",
 886                       help="Update the wiki")
 887     parser.add_option("", "--pretty", action="store_true", default=False,
 888                       help="Produce human-readable index.xml")
 889     parser.add_option("--clean", action="store_true", default=False,
 890                       help="Clean update - don't uses caches, reprocess all apks")
 891     (options, args) = parser.parse_args()
 892
 893     config = common.read_config(options)
 894
 895     repodirs = ['repo']
 896     if config['archive_older'] != 0:
 897         repodirs.append('archive')
 898         if not os.path.exists('archive'):
 899             os.mkdir('archive')
 900
 901     if options.icons:
 902         resize_all_icons(repodirs)
 903         sys.exit(0)
 904
 905     # Get all apps...
 906     apps = metadata.read_metadata()
 907
 908     # Generate a list of categories...
 909     categories = []
 910     for app in apps:
 911         cats = app['Categories'].split(',')
 912         for cat in cats:
 913             if cat not in categories:
 914                 categories.append(cat)
 915
 916     # Read known apks data (will be updated and written back when we've finished)
 917     knownapks = common.KnownApks()
 918
 919     # Gather information about all the apk files in the repo directory, using
 920     # cached data if possible.
 921     apkcachefile = os.path.join('tmp', 'apkcache')
 922     if not options.clean and os.path.exists(apkcachefile):
 923         with open(apkcachefile, 'rb') as cf:
 924             apkcache = pickle.load(cf)
 925     else:
 926         apkcache = {}
 927     cachechanged = False
 928
 929     delete_disabled_builds(apps, apkcache, repodirs)
 930
 931     # Scan all apks in the main repo
 932     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 933     if cc:
 934         cachechanged = True
 935
 936     # Scan the archive repo for apks as well
 937     if len(repodirs) > 1:
 938         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 939         if cc:
 940             cachechanged = True
 941     else:
 942         archapks = []
 943
 944     # Some information from the apks needs to be applied up to the application
 945     # level. When doing this, we use the info from the most recent version's apk.
 946     # We deal with figuring out when the app was added and last updated at the
 947     # same time.
 948     for app in apps:
 949         bestver = 0
 950         added = None
 951         lastupdated = None
 952         for apk in apks + archapks:
 953             if apk['id'] == app['id']:
 954                 if apk['versioncode'] > bestver:
 955                     bestver = apk['versioncode']
 956                     bestapk = apk
 957
 958                 if 'added' in apk:
 959                     if not added or apk['added'] < added:
 960                         added = apk['added']
 961                     if not lastupdated or apk['added'] > lastupdated:
 962                         lastupdated = apk['added']
 963
 964         if added:
 965             app['added'] = added
 966         else:
 967             if options.verbose:
 968                 print "WARNING: Don't know when " + app['id'] + " was added"
 969         if lastupdated:
 970             app['lastupdated'] = lastupdated
 971         else:
 972             if options.verbose:
 973                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 974
 975         if bestver == 0:
 976             if app['Name'] is None:
 977                 app['Name'] = app['id']
 978             app['icon'] = None
 979             if options.verbose and app['Disabled'] is None:
 980                 print "WARNING: Application " + app['id'] + " has no packages"
 981         else:
 982             if app['Name'] is None:
 983                 app['Name'] = bestapk['name']
 984             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 985
 986     # Sort the app list by name, then the web site doesn't have to by default.
 987     # (we had to wait until we'd scanned the apks to do this, because mostly the
 988     # name comes from there!)
 989     apps = sorted(apps, key=lambda app: app['Name'].upper())
 990
 991     # Generate warnings for apk's with no metadata (or create skeleton
 992     # metadata files, if requested on the command line)
 993     for apk in apks:
 994         found = False
 995         for app in apps:
 996             if app['id'] == apk['id']:
 997                 found = True
 998                 break
 999         if not found:
1000             if options.createmeta:
1001                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1002                 f.write("License:Unknown\n")
1003                 f.write("Web Site:\n")
1004                 f.write("Source Code:\n")
1005                 f.write("Issue Tracker:\n")
1006                 f.write("Summary:" + apk['name'] + "\n")
1007                 f.write("Description:\n")
1008                 f.write(apk['name'] + "\n")
1009                 f.write(".\n")
1010                 f.close()
1011                 print "Generated skeleton metadata for " + apk['id']
1012             else:
1013                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
1014                 print "       " + apk['name'] + " - " + apk['version']
1015
1016     if len(repodirs) > 1:
1017         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1018
1019     # Make the index for the main repo...
1020     make_index(apps, apks, repodirs[0], False, categories)
1021
1022     # If there's an archive repo,  make the index for it. We already scanned it
1023     # earlier on.
1024     if len(repodirs) > 1:
1025         make_index(apps, archapks, repodirs[1], True, categories)
1026
1027     if config['update_stats']:
1028
1029         # Update known apks info...
1030         knownapks.writeifchanged()
1031
1032         # Generate latest apps data for widget
1033         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1034             data = ''
1035             for line in file(os.path.join('stats', 'latestapps.txt')):
1036                 appid = line.rstrip()
1037                 data += appid + "\t"
1038                 for app in apps:
1039                     if app['id'] == appid:
1040                         data += app['Name'] + "\t"
1041                         if app['icon'] is not None:
1042                             data += app['icon'] + "\t"
1043                         data += app['License'] + "\n"
1044                         break
1045             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1046             f.write(data)
1047             f.close()
1048
1049     if cachechanged:
1050         with open(apkcachefile, 'wb') as cf:
1051             pickle.dump(apkcache, cf)
1052
1053     # Update the wiki...
1054     if options.wiki:
1055         update_wiki(apps, apks + archapks)
1056
1057     print "Finished."
1058
1059 if __name__ == "__main__":
1060     main()
1061