fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['480', '320', '240', '160', '120']
  40
  41 def launcher_size(density):
  42     return (int(density) * 48) / 160
  43
  44 def icon_dens_label(density):
  45     return "icon-%s" % density
  46
  47 def get_icon_dir(repodir, density):
  48     if density is None:
  49         return os.path.join(repodir, "icons")
  50     return os.path.join(repodir, "icons-%s" % density)
  51
  52 def get_icon_dirs(repodir):
  53     for density in get_densities():
  54         yield get_icon_dir(repodir, density)
  55     yield os.path.join(repodir, "icons")
  56
  57 def update_wiki(apps, apks):
  58     """Update the wiki
  59
  60     :param apps: fully populated list of all applications
  61     :param apks: all apks, except...
  62     """
  63     print "Updating wiki"
  64     wikicat = 'Apps'
  65     wikiredircat = 'App Redirects'
  66     import mwclient
  67     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  68             path=config['wiki_path'])
  69     site.login(config['wiki_user'], config['wiki_password'])
  70     generated_pages = {}
  71     generated_redirects = {}
  72     for app in apps:
  73         wikidata = ''
  74         if app['Disabled']:
  75             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  76         if app['AntiFeatures']:
  77             for af in app['AntiFeatures'].split(','):
  78                 wikidata += '{{AntiFeature|' + af + '}}\n'
  79         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  80                 app['id'],
  81                 app['Name'],
  82                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  83                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  84                 app['Source Code'],
  85                 app['Issue Tracker'],
  86                 app['Web Site'],
  87                 app['Donate'],
  88                 app['FlattrID'],
  89                 app['Bitcoin'],
  90                 app['Litecoin'],
  91                 app['Dogecoin'],
  92                 app['License'],
  93                 app.get('Requires Root', 'No'))
  94
  95         if app['Provides']:
  96             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  97
  98         wikidata += app['Summary']
  99         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 100
 101         wikidata += "=Description=\n"
 102         wikidata += metadata.description_wiki(app['Description']) + "\n"
 103
 104         wikidata += "=Maintainer Notes=\n"
 105         if 'Maintainer Notes' in app:
 106             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 107         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 108
 109         # Get a list of all packages for this application...
 110         apklist = []
 111         gotcurrentver = False
 112         cantupdate = False
 113         buildfails = False
 114         for apk in apks:
 115             if apk['id'] == app['id']:
 116                 if str(apk['versioncode']) == app['Current Version Code']:
 117                     gotcurrentver = True
 118                 apklist.append(apk)
 119         # Include ones we can't build, as a special case...
 120         for thisbuild in app['builds']:
 121             if 'disable' in thisbuild:
 122                 if thisbuild['vercode'] == app['Current Version Code']:
 123                     cantupdate = True
 124                 apklist.append({
 125                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 126                         'versioncode': int(thisbuild['vercode']),
 127                         'version': thisbuild['version'],
 128                         'buildproblem': thisbuild['disable']
 129                     })
 130             else:
 131                 builtit = False
 132                 for apk in apklist:
 133                     if apk['versioncode'] == int(thisbuild['vercode']):
 134                         builtit = True
 135                         break
 136                 if not builtit:
 137                     buildfails = True
 138                     apklist.append({
 139                             'versioncode': int(thisbuild['vercode']),
 140                             'version': thisbuild['version'],
 141                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 142                         })
 143         # Sort with most recent first...
 144         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 145
 146         wikidata += "=Versions=\n"
 147         if len(apklist) == 0:
 148             wikidata += "We currently have no versions of this app available."
 149         elif not gotcurrentver:
 150             wikidata += "We don't have the current version of this app."
 151         else:
 152             wikidata += "We have the current version of this app."
 153         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 154         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 155         if len(app['No Source Since']) > 0:
 156             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 157         if len(app['Current Version']) > 0:
 158             wikidata += "The current (recommended) version is " + app['Current Version']
 159             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 160         validapks = 0
 161         for apk in apklist:
 162             wikidata += "==" + apk['version'] + "==\n"
 163
 164             if 'buildproblem' in apk:
 165                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 166             else:
 167                 validapks += 1
 168                 wikidata += "This version is built and signed by "
 169                 if 'srcname' in apk:
 170                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 171                 else:
 172                     wikidata += "the original developer.\n\n"
 173             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 174
 175         wikidata += '\n[[Category:' + wikicat + ']]\n'
 176         if len(app['No Source Since']) > 0:
 177             wikidata += '\n[[Category:Apps missing source code]]\n'
 178         elif validapks == 0 and not app['Disabled']:
 179             wikidata += '\n[[Category:Apps with no packages]]\n'
 180         elif cantupdate and not app['Disabled']:
 181             wikidata += "\n[[Category:Apps we can't update]]\n"
 182         elif buildfails and not app['Disabled']:
 183             wikidata += "\n[[Category:Apps with failing builds]]\n"
 184         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 185             wikidata += '\n[[Category:Apps to Update]]\n'
 186         if app['Disabled']:
 187             wikidata += '\n[[Category:Apps that are disabled]]\n'
 188         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no update check]]\n'
 190
 191         # We can't have underscores in the page name, even if they're in
 192         # the package ID, because MediaWiki messes with them...
 193         pagename = app['id'].replace('_', ' ')
 194
 195         # Drop a trailing newline, because mediawiki is going to drop it anyway
 196         # and it we don't we'll think the page has changed when it hasn't...
 197         if wikidata.endswith('\n'):
 198             wikidata = wikidata[:-1]
 199
 200         generated_pages[pagename] = wikidata
 201
 202         # Make a redirect from the name to the ID too, unless there's
 203         # already an existing page with the name and it isn't a redirect.
 204         noclobber = False
 205         apppagename = app['Name'].replace('_', ' ')
 206         apppagename = apppagename.replace('{', '')
 207         apppagename = apppagename.replace('}', ' ')
 208         apppagename = apppagename.replace(':', ' ')
 209         # Drop double spaces caused mostly by replacing ':' above
 210         apppagename = apppagename.replace('  ', ' ')
 211         for expagename in site.allpages(prefix=apppagename,
 212                 filterredir='nonredirects', generator=False):
 213             if expagename == apppagename:
 214                 noclobber = True
 215         # Another reason not to make the redirect page is if the app name
 216         # is the same as it's ID, because that will overwrite the real page
 217         # with an redirect to itself! (Although it seems like an odd
 218         # scenario this happens a lot, e.g. where there is metadata but no
 219         # builds or binaries to extract a name from.
 220         if apppagename == pagename:
 221             noclobber = True
 222         if not noclobber:
 223             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 224
 225     for tcat, genp in [(wikicat, generated_pages),
 226             (wikiredircat, generated_redirects)]:
 227         catpages = site.Pages['Category:' + tcat]
 228         existingpages = []
 229         for page in catpages:
 230             existingpages.append(page.name)
 231             if page.name in genp:
 232                 pagetxt = page.edit()
 233                 if pagetxt != genp[page.name]:
 234                     print "Updating modified page " + page.name
 235                     page.save(genp[page.name], summary='Auto-updated')
 236                 else:
 237                     if options.verbose:
 238                         print "Page " + page.name + " is unchanged"
 239             else:
 240                 print "Deleting page " + page.name
 241                 page.delete('No longer published')
 242         for pagename, text in genp.items():
 243             if options.verbose:
 244                 print "Checking " + pagename
 245             if not pagename in existingpages:
 246                 print "Creating page " + pagename
 247                 try:
 248                     newpage = site.Pages[pagename]
 249                     newpage.save(text, summary='Auto-created')
 250                 except:
 251                     print "...FAILED to create page"
 252
 253     # Purge server cache to ensure counts are up to date
 254     site.pages['Repository Maintenance'].purge()
 255
 256 def delete_disabled_builds(apps, apkcache, repodirs):
 257     """Delete disabled build outputs.
 258
 259     :param apps: list of all applications, as per metadata.read_metadata
 260     :param apkcache: current apk cache information
 261     :param repodirs: the repo directories to process
 262     """
 263     for app in apps:
 264         for build in app['builds']:
 265             if 'disable' in build:
 266                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 267                 for repodir in repodirs:
 268                     apkpath = os.path.join(repodir, apkfilename)
 269                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 270                     for name in [apkpath, srcpath]:
 271                         if os.path.exists(name):
 272                             print "Deleting disabled build output " + apkfilename
 273                             os.remove(name)
 274                 if apkfilename in apkcache:
 275                     del apkcache[apkfilename]
 276
 277 def resize_icon(iconpath, density):
 278
 279     try:
 280         im = Image.open(iconpath)
 281         size = launcher_size(density)
 282
 283         if any(length > size for length in im.size):
 284             print iconpath, "is too large:", im.size
 285             im.thumbnail((size, size), Image.ANTIALIAS)
 286             print iconpath, "new size:", im.size
 287             im.save(iconpath, "PNG")
 288
 289         else:
 290             if options.verbose:
 291                 print iconpath, "is small enough:", im.size
 292
 293     except Exception,e:
 294         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 295
 296 def resize_all_icons(repodirs):
 297     """Resize all icons that exceed the max size
 298
 299     :param repodirs: the repo directories to process
 300     """
 301     for repodir in repodirs:
 302         for density in get_densities():
 303             icon_dir = get_icon_dir(repodir, density)
 304             icon_glob = os.path.join(repodir, icon_dir, '*.png')
 305             for iconpath in glob.glob(icon_glob):
 306                 resize_icon(iconpath)
 307
 308 def scan_apks(apps, apkcache, repodir, knownapks):
 309     """Scan the apks in the given repo directory.
 310
 311     This also extracts the icons.
 312
 313     :param apps: list of all applications, as per metadata.read_metadata
 314     :param apkcache: current apk cache information
 315     :param repodir: repo directory to scan
 316     :param knownapks: known apks info
 317     :returns: (apks, cachechanged) where apks is a list of apk information,
 318               and cachechanged is True if the apkcache got changed.
 319     """
 320
 321     cachechanged = False
 322
 323     icon_dirs = get_icon_dirs(repodir)
 324     for icon_dir in icon_dirs:
 325         if os.path.exists(icon_dir):
 326             if options.clean:
 327                 shutil.rmtree(icon_dir)
 328                 os.makedirs(icon_dir)
 329         else:
 330             os.makedirs(icon_dir)
 331
 332     apks = []
 333     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 334     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 335     vername_pat = re.compile(".*versionName='([^']*)'.*")
 336     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 337     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 338     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 339     string_pat = re.compile(".*'([^']*)'.*")
 340     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 341
 342         apkfilename = apkfile[len(repodir) + 1:]
 343         if ' ' in apkfilename:
 344             print "No spaces in APK filenames!"
 345             sys.exit(1)
 346
 347         if apkfilename in apkcache:
 348             if options.verbose:
 349                 print "Reading " + apkfilename + " from cache"
 350             thisinfo = apkcache[apkfilename]
 351
 352         else:
 353
 354             if not options.quiet:
 355                 print "Processing " + apkfilename
 356             thisinfo = {}
 357             thisinfo['apkname'] = apkfilename
 358             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 359             if os.path.exists(os.path.join(repodir, srcfilename)):
 360                 thisinfo['srcname'] = srcfilename
 361             thisinfo['size'] = os.path.getsize(apkfile)
 362             thisinfo['permissions'] = []
 363             thisinfo['features'] = []
 364             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 365                                   'dump', 'badging', apkfile],
 366                                  stdout=subprocess.PIPE)
 367             output = p.communicate()[0]
 368             #if options.verbose:
 369                 #print output
 370             if p.returncode != 0:
 371                 print "ERROR: Failed to get apk information"
 372                 sys.exit(1)
 373             for line in output.splitlines():
 374                 if line.startswith("package:"):
 375                     try:
 376                         thisinfo['id'] = re.match(name_pat, line).group(1)
 377                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 378                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 379                     except Exception, e:
 380                         print "Package matching failed: " + str(e)
 381                         print "Line was: " + line
 382                         sys.exit(1)
 383                 elif line.startswith("application:"):
 384                     thisinfo['name'] = re.match(label_pat, line).group(1)
 385                 elif line.startswith("application-icon-"):
 386                     match = re.match(icon_pat, line)
 387                     if match:
 388                         density = match.group(1)
 389                         path = match.group(2)
 390                         if 'icons_src' not in thisinfo:
 391                             thisinfo['icons_src'] = {}
 392                         thisinfo['icons_src'][density] = path
 393                 elif line.startswith("sdkVersion:"):
 394                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 395                 elif line.startswith("native-code:"):
 396                     thisinfo['nativecode'] = []
 397                     for arch in line[13:].split(' '):
 398                         thisinfo['nativecode'].append(arch[1:-1])
 399                 elif line.startswith("uses-permission:"):
 400                     perm = re.match(string_pat, line).group(1)
 401                     if perm.startswith("android.permission."):
 402                         perm = perm[19:]
 403                     thisinfo['permissions'].append(perm)
 404                 elif line.startswith("uses-feature:"):
 405                     perm = re.match(string_pat, line).group(1)
 406                     #Filter out this, it's only added with the latest SDK tools and
 407                     #causes problems for lots of apps.
 408                     if (perm != "android.hardware.screen.portrait" and
 409                         perm != "android.hardware.screen.landscape"):
 410                         if perm.startswith("android.feature."):
 411                             perm = perm[16:]
 412                         thisinfo['features'].append(perm)
 413
 414             if not 'sdkversion' in thisinfo:
 415                 print "  WARNING: no SDK version information found"
 416                 thisinfo['sdkversion'] = 0
 417
 418             # Check for debuggable apks...
 419             if common.isApkDebuggable(apkfile, config):
 420                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 421
 422             # Calculate the sha256...
 423             sha = hashlib.sha256()
 424             with open(apkfile, 'rb') as f:
 425                 while True:
 426                     t = f.read(1024)
 427                     if len(t) == 0:
 428                         break
 429                     sha.update(t)
 430                 thisinfo['sha256'] = sha.hexdigest()
 431
 432             # Get the signature (or md5 of, to be precise)...
 433             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 434             if not os.path.exists(getsig_dir + "/getsig.class"):
 435                 print "ERROR: getsig.class not found. To fix:"
 436                 print "\tcd " + getsig_dir
 437                 print "\t./make.sh"
 438                 sys.exit(1)
 439             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 440                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 441             output = p.communicate()[0]
 442             if options.verbose:
 443                 print output
 444             if p.returncode != 0 or not output.startswith('Result:'):
 445                 print "ERROR: Failed to get apk signature"
 446                 sys.exit(1)
 447             thisinfo['sig'] = output[7:].strip()
 448
 449             iconfilename = "%s.%s.png" % (
 450                     thisinfo['id'],
 451                     thisinfo['versioncode'])
 452
 453             # Extract the icon file...
 454             densities = get_densities()
 455             empty_densities = []
 456             for density in densities:
 457                 label = icon_dens_label(density)
 458                 if density not in thisinfo['icons_src']:
 459                     empty_densities.append(density)
 460                     continue
 461                 apk = zipfile.ZipFile(apkfile, 'r')
 462                 iconsrc = thisinfo['icons_src'][density]
 463                 icon_dir = get_icon_dir(repodir, density)
 464                 icondest = os.path.join(icon_dir, iconfilename)
 465
 466                 try:
 467                     iconfile = open(icondest, 'wb')
 468                     iconfile.write(apk.read(iconsrc))
 469                     iconfile.close()
 470                     thisinfo[label] = iconfilename
 471
 472                 except:
 473                     print "WARNING: Error retrieving icon file"
 474                     del thisinfo[label]
 475                     del thisinfo['icons_src'][density]
 476                     empty_densities.append(density)
 477
 478                 apk.close()
 479
 480                 resize_icon(icondest, density)
 481
 482             last_density = None
 483             # First try resizing down to not lose quality
 484             for density in densities:
 485                 if density not in empty_densities:
 486                     last_density = density
 487                     continue
 488                 if last_density is None:
 489                     continue
 490                 if options.verbose:
 491                     print "Density %s not available, resizing down from %s" % (
 492                             density, last_density)
 493
 494                 last_iconpath = os.path.join(
 495                         get_icon_dir(repodir, last_density), iconfilename)
 496                 iconpath = os.path.join(
 497                         get_icon_dir(repodir, density), iconfilename)
 498                 im = Image.open(last_iconpath)
 499                 size = launcher_size(density)
 500
 501                 im.thumbnail((size, size), Image.ANTIALIAS)
 502                 im.save(iconpath, "PNG")
 503                 empty_densities.remove(density)
 504
 505             last_density = None
 506             # Then just copy from the highest resolution available
 507             for density in reversed(densities):
 508                 if density not in empty_densities:
 509                     last_density = density
 510                     continue
 511                 if last_density is None:
 512                     continue
 513                 if options.verbose:
 514                     print "Density %s not available, copying from lower density %s" % (
 515                             density, last_density)
 516
 517                 shutil.copyfile(
 518                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 519                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 520
 521                 empty_densities.remove(density)
 522
 523             # Copy from icons-mdpi to icons since mdpi is the baseline density
 524             shutil.copyfile(
 525                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 526                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 527
 528             # Record in known apks, getting the added date at the same time..
 529             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 530             if added:
 531                 thisinfo['added'] = added
 532
 533             apkcache[apkfilename] = thisinfo
 534             cachechanged = True
 535
 536         apks.append(thisinfo)
 537
 538     return apks, cachechanged
 539
 540
 541 repo_pubkey_fingerprint = None
 542
 543 def make_index(apps, apks, repodir, archive, categories):
 544     """Make a repo index.
 545
 546     :param apps: fully populated apps list
 547     :param apks: full populated apks list
 548     :param repodir: the repo directory
 549     :param archive: True if this is the archive repo, False if it's the
 550                     main one.
 551     :param categories: list of categories
 552     """
 553
 554     doc = Document()
 555
 556     def addElement(name, value, doc, parent):
 557         el = doc.createElement(name)
 558         el.appendChild(doc.createTextNode(value))
 559         parent.appendChild(el)
 560     def addElementCDATA(name, value, doc, parent):
 561         el = doc.createElement(name)
 562         el.appendChild(doc.createCDATASection(value))
 563         parent.appendChild(el)
 564
 565     root = doc.createElement("fdroid")
 566     doc.appendChild(root)
 567
 568     repoel = doc.createElement("repo")
 569
 570     if archive:
 571         repoel.setAttribute("name", config['archive_name'])
 572         if config['repo_maxage'] != 0:
 573             repoel.setAttribute("maxage", str(config['repo_maxage']))
 574         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 575         repoel.setAttribute("url", config['archive_url'])
 576         addElement('description', config['archive_description'], doc, repoel)
 577
 578     else:
 579         repoel.setAttribute("name", config['repo_name'])
 580         if config['repo_maxage'] != 0:
 581             repoel.setAttribute("maxage", str(config['repo_maxage']))
 582         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 583         repoel.setAttribute("url", config['repo_url'])
 584         addElement('description', config['repo_description'], doc, repoel)
 585
 586     repoel.setAttribute("version", "10")
 587     repoel.setAttribute("timestamp", str(int(time.time())))
 588
 589     if config['repo_keyalias']:
 590
 591         # Generate a certificate fingerprint the same way keytool does it
 592         # (but with slightly different formatting)
 593         def cert_fingerprint(data):
 594             digest = hashlib.sha1(data).digest()
 595             ret = []
 596             for i in range(4):
 597                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 598             return " ".join(ret)
 599
 600         def extract_pubkey():
 601             p = subprocess.Popen(['keytool', '-exportcert',
 602                                   '-alias', config['repo_keyalias'],
 603                                   '-keystore', config['keystore'],
 604                                   '-storepass', config['keystorepass']],
 605                                  stdout=subprocess.PIPE)
 606             cert = p.communicate()[0]
 607             if p.returncode != 0:
 608                 print "ERROR: Failed to get repo pubkey"
 609                 sys.exit(1)
 610             global repo_pubkey_fingerprint
 611             repo_pubkey_fingerprint = cert_fingerprint(cert)
 612             return "".join("%02x" % ord(b) for b in cert)
 613
 614         repoel.setAttribute("pubkey", extract_pubkey())
 615
 616     root.appendChild(repoel)
 617
 618     for app in apps:
 619
 620         if app['Disabled'] is not None:
 621             continue
 622
 623         # Get a list of the apks for this app...
 624         apklist = []
 625         for apk in apks:
 626             if apk['id'] == app['id']:
 627                 apklist.append(apk)
 628
 629         if len(apklist) == 0:
 630             continue
 631
 632         apel = doc.createElement("application")
 633         apel.setAttribute("id", app['id'])
 634         root.appendChild(apel)
 635
 636         addElement('id', app['id'], doc, apel)
 637         if 'added' in app:
 638             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 639         if 'lastupdated' in app:
 640             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 641         addElement('name', app['Name'], doc, apel)
 642         addElement('summary', app['Summary'], doc, apel)
 643         if app['icon']:
 644             addElement('icon', app['icon'], doc, apel)
 645         def linkres(link):
 646             for app in apps:
 647                 if app['id'] == link:
 648                     return ("fdroid.app:" + link, app['Name'])
 649             raise MetaDataException("Cannot resolve app id " + link)
 650         addElement('desc',
 651                 metadata.description_html(app['Description'], linkres), doc, apel)
 652         addElement('license', app['License'], doc, apel)
 653         if 'Categories' in app:
 654             appcategories = [c.strip() for c in app['Categories'].split(',')]
 655             addElement('categories', ','.join(appcategories), doc, apel)
 656             # We put the first (primary) category in LAST, which will have
 657             # the desired effect of making clients that only understand one
 658             # category see that one.
 659             addElement('category', appcategories[0], doc, apel)
 660         addElement('web', app['Web Site'], doc, apel)
 661         addElement('source', app['Source Code'], doc, apel)
 662         addElement('tracker', app['Issue Tracker'], doc, apel)
 663         if app['Donate']:
 664             addElement('donate', app['Donate'], doc, apel)
 665         if app['Bitcoin']:
 666             addElement('bitcoin', app['Bitcoin'], doc, apel)
 667         if app['Litecoin']:
 668             addElement('litecoin', app['Litecoin'], doc, apel)
 669         if app['Dogecoin']:
 670             addElement('dogecoin', app['Dogecoin'], doc, apel)
 671         if app['FlattrID']:
 672             addElement('flattr', app['FlattrID'], doc, apel)
 673
 674         # These elements actually refer to the current version (i.e. which
 675         # one is recommended. They are historically mis-named, and need
 676         # changing, but stay like this for now to support existing clients.
 677         addElement('marketversion', app['Current Version'], doc, apel)
 678         addElement('marketvercode', app['Current Version Code'], doc, apel)
 679
 680         if app['AntiFeatures']:
 681             af = app['AntiFeatures'].split(',')
 682             # TODO: Temporarily not including UpstreamNonFree in the index,
 683             # because current F-Droid clients do not understand it, and also
 684             # look ugly when they encounter an unknown antifeature. This
 685             # filtering can be removed in time...
 686             if 'UpstreamNonFree' in af:
 687                 af.remove('UpstreamNonFree')
 688             if af:
 689                 addElement('antifeatures', ','.join(af), doc, apel)
 690         if app['Provides']:
 691             pv = app['Provides'].split(',')
 692             addElement('provides', ','.join(pv), doc, apel)
 693         if app['Requires Root']:
 694             addElement('requirements', 'root', doc, apel)
 695
 696         # Sort the apk list into version order, just so the web site
 697         # doesn't have to do any work by default...
 698         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 699
 700         # Check for duplicates - they will make the client unhappy...
 701         for i in range(len(apklist) - 1):
 702             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 703                 print "ERROR - duplicate versions"
 704                 print apklist[i]['apkname']
 705                 print apklist[i+1]['apkname']
 706                 sys.exit(1)
 707
 708         for apk in apklist:
 709             apkel = doc.createElement("package")
 710             apel.appendChild(apkel)
 711             addElement('version', apk['version'], doc, apkel)
 712             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 713             addElement('apkname', apk['apkname'], doc, apkel)
 714             if 'srcname' in apk:
 715                 addElement('srcname', apk['srcname'], doc, apkel)
 716             for hash_type in ['sha256']:
 717                 if not hash_type in apk:
 718                     continue
 719                 hashel = doc.createElement("hash")
 720                 hashel.setAttribute("type", hash_type)
 721                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 722                 apkel.appendChild(hashel)
 723             addElement('sig', apk['sig'], doc, apkel)
 724             addElement('size', str(apk['size']), doc, apkel)
 725             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 726             if 'added' in apk:
 727                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 728             if app['Requires Root']:
 729                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 730                     apk['permissions'].append('ACCESS_SUPERUSER')
 731
 732             if len(apk['permissions']) > 0:
 733                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 734             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 735                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 736             if len(apk['features']) > 0:
 737                 addElement('features', ','.join(apk['features']), doc, apkel)
 738
 739     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 740     if options.pretty:
 741         output = doc.toprettyxml()
 742     else:
 743         output = doc.toxml()
 744     of.write(output)
 745     of.close()
 746
 747     if config['repo_keyalias'] is not None:
 748
 749         if not options.quiet:
 750             print "Creating signed index."
 751             print "Key fingerprint:", repo_pubkey_fingerprint
 752
 753         #Create a jar of the index...
 754         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 755             cwd=repodir, stdout=subprocess.PIPE)
 756         output = p.communicate()[0]
 757         if options.verbose:
 758             print output
 759         if p.returncode != 0:
 760             print "ERROR: Failed to create jar file"
 761             sys.exit(1)
 762
 763         # Sign the index...
 764         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 765             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 766             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 767             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 768         output = p.communicate()[0]
 769         if p.returncode != 0:
 770             print "Failed to sign index"
 771             print output
 772             sys.exit(1)
 773         if options.verbose:
 774             print output
 775
 776     # Copy the repo icon into the repo directory...
 777     icon_dir = os.path.join(repodir ,'icons')
 778     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 779     shutil.copyfile(config['repo_icon'], iconfilename)
 780
 781     # Write a category list in the repo to allow quick access...
 782     catdata = ''
 783     for cat in categories:
 784         catdata += cat + '\n'
 785     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 786     f.write(catdata)
 787     f.close()
 788
 789
 790
 791 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 792
 793     for app in apps:
 794
 795         # Get a list of the apks for this app...
 796         apklist = []
 797         for apk in apks:
 798             if apk['id'] == app['id']:
 799                 apklist.append(apk)
 800
 801         # Sort the apk list into version order...
 802         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 803
 804         if app['Archive Policy']:
 805             keepversions = int(app['Archive Policy'][:-9])
 806         else:
 807             keepversions = defaultkeepversions
 808
 809         if len(apklist) > keepversions:
 810             for apk in apklist[keepversions:]:
 811                 print "Moving " + apk['apkname'] + " to archive"
 812                 shutil.move(os.path.join(repodir, apk['apkname']),
 813                     os.path.join(archivedir, apk['apkname']))
 814                 if 'srcname' in apk:
 815                     shutil.move(os.path.join(repodir, apk['srcname']),
 816                         os.path.join(archivedir, apk['srcname']))
 817                 apks.remove(apk)
 818
 819
 820 config = None
 821 options = None
 822
 823 def main():
 824
 825     global config, options
 826
 827     # Parse command line...
 828     parser = OptionParser()
 829     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 830                       help="Create skeleton metadata files that are missing")
 831     parser.add_option("-v", "--verbose", action="store_true", default=False,
 832                       help="Spew out even more information than normal")
 833     parser.add_option("-q", "--quiet", action="store_true", default=False,
 834                       help="No output, except for warnings and errors")
 835     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 836                       help="Report on build data status")
 837     parser.add_option("-i", "--interactive", default=False, action="store_true",
 838                       help="Interactively ask about things that need updating.")
 839     parser.add_option("-I", "--icons", action="store_true", default=False,
 840                       help="Resize all the icons exceeding the max pixel size and exit")
 841     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 842                       help="Specify editor to use in interactive mode. Default "+
 843                           "is /etc/alternatives/editor")
 844     parser.add_option("-w", "--wiki", default=False, action="store_true",
 845                       help="Update the wiki")
 846     parser.add_option("", "--pretty", action="store_true", default=False,
 847                       help="Produce human-readable index.xml")
 848     parser.add_option("--clean", action="store_true", default=False,
 849                       help="Clean update - don't uses caches, reprocess all apks")
 850     (options, args) = parser.parse_args()
 851
 852     config = common.read_config(options)
 853
 854     repodirs = ['repo']
 855     if config['archive_older'] != 0:
 856         repodirs.append('archive')
 857         if not os.path.exists('archive'):
 858             os.mkdir('archive')
 859
 860     if options.icons:
 861         resize_all_icons(repodirs)
 862         sys.exit(0)
 863
 864     # Get all apps...
 865     apps = metadata.read_metadata()
 866
 867     # Generate a list of categories...
 868     categories = []
 869     for app in apps:
 870         cats = app['Categories'].split(',')
 871         for cat in cats:
 872             if cat not in categories:
 873                 categories.append(cat)
 874
 875     # Read known apks data (will be updated and written back when we've finished)
 876     knownapks = common.KnownApks()
 877
 878     # Gather information about all the apk files in the repo directory, using
 879     # cached data if possible.
 880     apkcachefile = os.path.join('tmp', 'apkcache')
 881     if not options.clean and os.path.exists(apkcachefile):
 882         with open(apkcachefile, 'rb') as cf:
 883             apkcache = pickle.load(cf)
 884     else:
 885         apkcache = {}
 886     cachechanged = False
 887
 888     delete_disabled_builds(apps, apkcache, repodirs)
 889
 890     # Scan all apks in the main repo
 891     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 892     if cc:
 893         cachechanged = True
 894
 895     # Scan the archive repo for apks as well
 896     if len(repodirs) > 1:
 897         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 898         if cc:
 899             cachechanged = True
 900     else:
 901         archapks = []
 902
 903     # Some information from the apks needs to be applied up to the application
 904     # level. When doing this, we use the info from the most recent version's apk.
 905     # We deal with figuring out when the app was added and last updated at the
 906     # same time.
 907     for app in apps:
 908         bestver = 0
 909         added = None
 910         lastupdated = None
 911         for apk in apks + archapks:
 912             if apk['id'] == app['id']:
 913                 if apk['versioncode'] > bestver:
 914                     bestver = apk['versioncode']
 915                     bestapk = apk
 916
 917                 if 'added' in apk:
 918                     if not added or apk['added'] < added:
 919                         added = apk['added']
 920                     if not lastupdated or apk['added'] > lastupdated:
 921                         lastupdated = apk['added']
 922
 923         if added:
 924             app['added'] = added
 925         #else:
 926             #if options.verbose:
 927                 #print "WARNING: Don't know when " + app['id'] + " was added"
 928         if lastupdated:
 929             app['lastupdated'] = lastupdated
 930         #else:
 931             #if options.verbose:
 932                 #print "WARNING: Don't know when " + app['id'] + " was last updated"
 933
 934         if bestver == 0:
 935             if app['Name'] is None:
 936                 app['Name'] = app['id']
 937             app['icon'] = None
 938             #if options.verbose and app['Disabled'] is None:
 939                 #print "WARNING: Application " + app['id'] + " has no packages"
 940         else:
 941             if app['Name'] is None:
 942                 app['Name'] = bestapk['name']
 943             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 944
 945     # Sort the app list by name, then the web site doesn't have to by default.
 946     # (we had to wait until we'd scanned the apks to do this, because mostly the
 947     # name comes from there!)
 948     apps = sorted(apps, key=lambda app: app['Name'].upper())
 949
 950     # Generate warnings for apk's with no metadata (or create skeleton
 951     # metadata files, if requested on the command line)
 952     for apk in apks:
 953         found = False
 954         for app in apps:
 955             if app['id'] == apk['id']:
 956                 found = True
 957                 break
 958         if not found:
 959             if options.createmeta:
 960                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 961                 f.write("License:Unknown\n")
 962                 f.write("Web Site:\n")
 963                 f.write("Source Code:\n")
 964                 f.write("Issue Tracker:\n")
 965                 f.write("Summary:" + apk['name'] + "\n")
 966                 f.write("Description:\n")
 967                 f.write(apk['name'] + "\n")
 968                 f.write(".\n")
 969                 f.close()
 970                 print "Generated skeleton metadata for " + apk['id']
 971             else:
 972                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 973                 print "       " + apk['name'] + " - " + apk['version']
 974
 975     if len(repodirs) > 1:
 976         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 977
 978     # Make the index for the main repo...
 979     make_index(apps, apks, repodirs[0], False, categories)
 980
 981     # If there's an archive repo,  make the index for it. We already scanned it
 982     # earlier on.
 983     if len(repodirs) > 1:
 984         make_index(apps, archapks, repodirs[1], True, categories)
 985
 986     if config['update_stats']:
 987
 988         # Update known apks info...
 989         knownapks.writeifchanged()
 990
 991         # Generate latest apps data for widget
 992         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 993             data = ''
 994             for line in file(os.path.join('stats', 'latestapps.txt')):
 995                 appid = line.rstrip()
 996                 data += appid + "\t"
 997                 for app in apps:
 998                     if app['id'] == appid:
 999                         data += app['Name'] + "\t"
1000                         if app['icon'] is not None:
1001                             data += app['icon'] + "\t"
1002                         data += app['License'] + "\n"
1003                         break
1004             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1005             f.write(data)
1006             f.close()
1007
1008     if cachechanged:
1009         with open(apkcachefile, 'wb') as cf:
1010             pickle.dump(apkcache, cf)
1011
1012     # Update the wiki...
1013     if options.wiki:
1014         update_wiki(apps, apks + archapks)
1015
1016     print "Finished."
1017
1018 if __name__ == "__main__":
1019     main()
1020