fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37
  38 def get_densities():
  39     return ['640', '480', '320', '240', '160', '120']
  40
  41 def launcher_size(density):
  42     return (int(density) * 48) / 160
  43
  44 def get_icon_dir(repodir, density):
  45     if density is None:
  46         return os.path.join(repodir, "icons")
  47     return os.path.join(repodir, "icons-%s" % density)
  48
  49 def get_icon_dirs(repodir):
  50     for density in get_densities():
  51         yield get_icon_dir(repodir, density)
  52     yield os.path.join(repodir, "icons")
  53
  54 def update_wiki(apps, apks):
  55     """Update the wiki
  56
  57     :param apps: fully populated list of all applications
  58     :param apks: all apks, except...
  59     """
  60     print "Updating wiki"
  61     wikicat = 'Apps'
  62     wikiredircat = 'App Redirects'
  63     import mwclient
  64     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  65             path=config['wiki_path'])
  66     site.login(config['wiki_user'], config['wiki_password'])
  67     generated_pages = {}
  68     generated_redirects = {}
  69     for app in apps:
  70         wikidata = ''
  71         if app['Disabled']:
  72             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  73         if app['AntiFeatures']:
  74             for af in app['AntiFeatures'].split(','):
  75                 wikidata += '{{AntiFeature|' + af + '}}\n'
  76         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  77                 app['id'],
  78                 app['Name'],
  79                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  80                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  81                 app['Source Code'],
  82                 app['Issue Tracker'],
  83                 app['Web Site'],
  84                 app['Donate'],
  85                 app['FlattrID'],
  86                 app['Bitcoin'],
  87                 app['Litecoin'],
  88                 app['Dogecoin'],
  89                 app['License'],
  90                 app.get('Requires Root', 'No'))
  91
  92         if app['Provides']:
  93             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  94
  95         wikidata += app['Summary']
  96         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  97
  98         wikidata += "=Description=\n"
  99         wikidata += metadata.description_wiki(app['Description']) + "\n"
 100
 101         wikidata += "=Maintainer Notes=\n"
 102         if 'Maintainer Notes' in app:
 103             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 104         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
 105
 106         # Get a list of all packages for this application...
 107         apklist = []
 108         gotcurrentver = False
 109         cantupdate = False
 110         buildfails = False
 111         for apk in apks:
 112             if apk['id'] == app['id']:
 113                 if str(apk['versioncode']) == app['Current Version Code']:
 114                     gotcurrentver = True
 115                 apklist.append(apk)
 116         # Include ones we can't build, as a special case...
 117         for thisbuild in app['builds']:
 118             if 'disable' in thisbuild:
 119                 if thisbuild['vercode'] == app['Current Version Code']:
 120                     cantupdate = True
 121                 apklist.append({
 122                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 123                         'versioncode': int(thisbuild['vercode']),
 124                         'version': thisbuild['version'],
 125                         'buildproblem': thisbuild['disable']
 126                     })
 127             else:
 128                 builtit = False
 129                 for apk in apklist:
 130                     if apk['versioncode'] == int(thisbuild['vercode']):
 131                         builtit = True
 132                         break
 133                 if not builtit:
 134                     buildfails = True
 135                     apklist.append({
 136                             'versioncode': int(thisbuild['vercode']),
 137                             'version': thisbuild['version'],
 138                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 139                         })
 140         # Sort with most recent first...
 141         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 142
 143         wikidata += "=Versions=\n"
 144         if len(apklist) == 0:
 145             wikidata += "We currently have no versions of this app available."
 146         elif not gotcurrentver:
 147             wikidata += "We don't have the current version of this app."
 148         else:
 149             wikidata += "We have the current version of this app."
 150         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 151         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 152         if len(app['No Source Since']) > 0:
 153             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 154         if len(app['Current Version']) > 0:
 155             wikidata += "The current (recommended) version is " + app['Current Version']
 156             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 157         validapks = 0
 158         for apk in apklist:
 159             wikidata += "==" + apk['version'] + "==\n"
 160
 161             if 'buildproblem' in apk:
 162                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 163             else:
 164                 validapks += 1
 165                 wikidata += "This version is built and signed by "
 166                 if 'srcname' in apk:
 167                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 168                 else:
 169                     wikidata += "the original developer.\n\n"
 170             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 171
 172         wikidata += '\n[[Category:' + wikicat + ']]\n'
 173         if len(app['No Source Since']) > 0:
 174             wikidata += '\n[[Category:Apps missing source code]]\n'
 175         elif validapks == 0 and not app['Disabled']:
 176             wikidata += '\n[[Category:Apps with no packages]]\n'
 177         elif cantupdate and not app['Disabled']:
 178             wikidata += "\n[[Category:Apps we can't update]]\n"
 179         elif buildfails and not app['Disabled']:
 180             wikidata += "\n[[Category:Apps with failing builds]]\n"
 181         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 182             wikidata += '\n[[Category:Apps to Update]]\n'
 183         if app['Disabled']:
 184             wikidata += '\n[[Category:Apps that are disabled]]\n'
 185         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 186             wikidata += '\n[[Category:Apps with no update check]]\n'
 187         for appcat in [c.strip() for c in app['Categories'].split(',')]:
 188             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 189
 190         # We can't have underscores in the page name, even if they're in
 191         # the package ID, because MediaWiki messes with them...
 192         pagename = app['id'].replace('_', ' ')
 193
 194         # Drop a trailing newline, because mediawiki is going to drop it anyway
 195         # and it we don't we'll think the page has changed when it hasn't...
 196         if wikidata.endswith('\n'):
 197             wikidata = wikidata[:-1]
 198
 199         generated_pages[pagename] = wikidata
 200
 201         # Make a redirect from the name to the ID too, unless there's
 202         # already an existing page with the name and it isn't a redirect.
 203         noclobber = False
 204         apppagename = app['Name'].replace('_', ' ')
 205         apppagename = apppagename.replace('{', '')
 206         apppagename = apppagename.replace('}', ' ')
 207         apppagename = apppagename.replace(':', ' ')
 208         # Drop double spaces caused mostly by replacing ':' above
 209         apppagename = apppagename.replace('  ', ' ')
 210         for expagename in site.allpages(prefix=apppagename,
 211                 filterredir='nonredirects', generator=False):
 212             if expagename == apppagename:
 213                 noclobber = True
 214         # Another reason not to make the redirect page is if the app name
 215         # is the same as it's ID, because that will overwrite the real page
 216         # with an redirect to itself! (Although it seems like an odd
 217         # scenario this happens a lot, e.g. where there is metadata but no
 218         # builds or binaries to extract a name from.
 219         if apppagename == pagename:
 220             noclobber = True
 221         if not noclobber:
 222             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 223
 224     for tcat, genp in [(wikicat, generated_pages),
 225             (wikiredircat, generated_redirects)]:
 226         catpages = site.Pages['Category:' + tcat]
 227         existingpages = []
 228         for page in catpages:
 229             existingpages.append(page.name)
 230             if page.name in genp:
 231                 pagetxt = page.edit()
 232                 if pagetxt != genp[page.name]:
 233                     print "Updating modified page " + page.name
 234                     page.save(genp[page.name], summary='Auto-updated')
 235                 else:
 236                     if options.verbose:
 237                         print "Page " + page.name + " is unchanged"
 238             else:
 239                 print "Deleting page " + page.name
 240                 page.delete('No longer published')
 241         for pagename, text in genp.items():
 242             if options.verbose:
 243                 print "Checking " + pagename
 244             if not pagename in existingpages:
 245                 print "Creating page " + pagename
 246                 try:
 247                     newpage = site.Pages[pagename]
 248                     newpage.save(text, summary='Auto-created')
 249                 except:
 250                     print "...FAILED to create page"
 251
 252     # Purge server cache to ensure counts are up to date
 253     site.pages['Repository Maintenance'].purge()
 254
 255 def delete_disabled_builds(apps, apkcache, repodirs):
 256     """Delete disabled build outputs.
 257
 258     :param apps: list of all applications, as per metadata.read_metadata
 259     :param apkcache: current apk cache information
 260     :param repodirs: the repo directories to process
 261     """
 262     for app in apps:
 263         for build in app['builds']:
 264             if 'disable' in build:
 265                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 266                 for repodir in repodirs:
 267                     apkpath = os.path.join(repodir, apkfilename)
 268                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 269                     for name in [apkpath, srcpath]:
 270                         if os.path.exists(name):
 271                             print "Deleting disabled build output " + apkfilename
 272                             os.remove(name)
 273                 if apkfilename in apkcache:
 274                     del apkcache[apkfilename]
 275
 276 def resize_icon(iconpath, density):
 277
 278     try:
 279         im = Image.open(iconpath)
 280         size = launcher_size(density)
 281
 282         if any(length > size for length in im.size):
 283             print iconpath, "is too large:", im.size
 284             im.thumbnail((size, size), Image.ANTIALIAS)
 285             print iconpath, "new size:", im.size
 286             im.save(iconpath, "PNG")
 287
 288         else:
 289             if options.verbose:
 290                 print iconpath, "is small enough:", im.size
 291
 292     except Exception,e:
 293         print "ERROR: Failed resizing {0} - {1}".format(iconpath, e)
 294
 295 def resize_all_icons(repodirs):
 296     """Resize all icons that exceed the max size
 297
 298     :param repodirs: the repo directories to process
 299     """
 300     for repodir in repodirs:
 301         for density in get_densities():
 302             icon_dir = get_icon_dir(repodir, density)
 303             icon_glob = os.path.join(icon_dir, '*.png')
 304             for iconpath in glob.glob(icon_glob):
 305                 resize_icon(iconpath, density)
 306
 307 def scan_apks(apps, apkcache, repodir, knownapks):
 308     """Scan the apks in the given repo directory.
 309
 310     This also extracts the icons.
 311
 312     :param apps: list of all applications, as per metadata.read_metadata
 313     :param apkcache: current apk cache information
 314     :param repodir: repo directory to scan
 315     :param knownapks: known apks info
 316     :returns: (apks, cachechanged) where apks is a list of apk information,
 317               and cachechanged is True if the apkcache got changed.
 318     """
 319
 320     cachechanged = False
 321
 322     icon_dirs = get_icon_dirs(repodir)
 323     for icon_dir in icon_dirs:
 324         if os.path.exists(icon_dir):
 325             if options.clean:
 326                 shutil.rmtree(icon_dir)
 327                 os.makedirs(icon_dir)
 328         else:
 329             os.makedirs(icon_dir)
 330
 331     apks = []
 332     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 333     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 334     vername_pat = re.compile(".*versionName='([^']*)'.*")
 335     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 336     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 337     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 338     string_pat = re.compile(".*'([^']*)'.*")
 339     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 340
 341         apkfilename = apkfile[len(repodir) + 1:]
 342         if ' ' in apkfilename:
 343             print "No spaces in APK filenames!"
 344             sys.exit(1)
 345
 346         if apkfilename in apkcache:
 347             if options.verbose:
 348                 print "Reading " + apkfilename + " from cache"
 349             thisinfo = apkcache[apkfilename]
 350
 351         else:
 352
 353             if not options.quiet:
 354                 print "Processing " + apkfilename
 355             thisinfo = {}
 356             thisinfo['apkname'] = apkfilename
 357             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 358             if os.path.exists(os.path.join(repodir, srcfilename)):
 359                 thisinfo['srcname'] = srcfilename
 360             thisinfo['size'] = os.path.getsize(apkfile)
 361             thisinfo['permissions'] = []
 362             thisinfo['features'] = []
 363             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 364                                   'dump', 'badging', apkfile],
 365                                  stdout=subprocess.PIPE)
 366             output = p.communicate()[0]
 367             if p.returncode != 0:
 368                 print "ERROR: Failed to get apk information"
 369                 sys.exit(1)
 370             for line in output.splitlines():
 371                 if line.startswith("package:"):
 372                     try:
 373                         thisinfo['id'] = re.match(name_pat, line).group(1)
 374                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 375                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 376                     except Exception, e:
 377                         print "Package matching failed: " + str(e)
 378                         print "Line was: " + line
 379                         sys.exit(1)
 380                 elif line.startswith("application:"):
 381                     thisinfo['name'] = re.match(label_pat, line).group(1)
 382                 elif line.startswith("application-icon-"):
 383                     match = re.match(icon_pat, line)
 384                     if match:
 385                         density = match.group(1)
 386                         path = match.group(2)
 387                         if 'icons_src' not in thisinfo:
 388                             thisinfo['icons_src'] = {}
 389                         thisinfo['icons_src'][density] = path
 390                 elif line.startswith("sdkVersion:"):
 391                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 392                 elif line.startswith("native-code:"):
 393                     thisinfo['nativecode'] = []
 394                     for arch in line[13:].split(' '):
 395                         thisinfo['nativecode'].append(arch[1:-1])
 396                 elif line.startswith("uses-permission:"):
 397                     perm = re.match(string_pat, line).group(1)
 398                     if perm.startswith("android.permission."):
 399                         perm = perm[19:]
 400                     thisinfo['permissions'].append(perm)
 401                 elif line.startswith("uses-feature:"):
 402                     perm = re.match(string_pat, line).group(1)
 403                     #Filter out this, it's only added with the latest SDK tools and
 404                     #causes problems for lots of apps.
 405                     if (perm != "android.hardware.screen.portrait" and
 406                         perm != "android.hardware.screen.landscape"):
 407                         if perm.startswith("android.feature."):
 408                             perm = perm[16:]
 409                         thisinfo['features'].append(perm)
 410
 411             if not 'sdkversion' in thisinfo:
 412                 print "  WARNING: no SDK version information found"
 413                 thisinfo['sdkversion'] = 0
 414
 415             # Check for debuggable apks...
 416             if common.isApkDebuggable(apkfile, config):
 417                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 418
 419             # Calculate the sha256...
 420             sha = hashlib.sha256()
 421             with open(apkfile, 'rb') as f:
 422                 while True:
 423                     t = f.read(1024)
 424                     if len(t) == 0:
 425                         break
 426                     sha.update(t)
 427                 thisinfo['sha256'] = sha.hexdigest()
 428
 429             # Get the signature (or md5 of, to be precise)...
 430             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 431             if not os.path.exists(getsig_dir + "/getsig.class"):
 432                 print "ERROR: getsig.class not found. To fix:"
 433                 print "\tcd " + getsig_dir
 434                 print "\t./make.sh"
 435                 sys.exit(1)
 436             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 437                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 438             output = p.communicate()[0]
 439             if options.verbose:
 440                 print output
 441             if p.returncode != 0 or not output.startswith('Result:'):
 442                 print "ERROR: Failed to get apk signature"
 443                 sys.exit(1)
 444             thisinfo['sig'] = output[7:].strip()
 445
 446             iconfilename = "%s.%s.png" % (
 447                     thisinfo['id'],
 448                     thisinfo['versioncode'])
 449
 450             # Extract the icon file...
 451             densities = get_densities()
 452             empty_densities = []
 453             for density in densities:
 454                 if density not in thisinfo['icons_src']:
 455                     empty_densities.append(density)
 456                     continue
 457                 apk = zipfile.ZipFile(apkfile, 'r')
 458                 if 'icons' not in thisinfo:
 459                     thisinfo['icons'] = {}
 460                 iconsrc = thisinfo['icons_src'][density]
 461                 icon_dir = get_icon_dir(repodir, density)
 462                 icondest = os.path.join(icon_dir, iconfilename)
 463
 464                 try:
 465                     iconfile = open(icondest, 'wb')
 466                     iconfile.write(apk.read(iconsrc))
 467                     iconfile.close()
 468                     thisinfo['icons'][density] = iconfilename
 469
 470                 except:
 471                     print "WARNING: Error retrieving icon file"
 472                     del thisinfo['icons'][density]
 473                     del thisinfo['icons_src'][density]
 474                     empty_densities.append(density)
 475
 476                 apk.close()
 477
 478                 resize_icon(icondest, density)
 479
 480             last_density = None
 481             # First try resizing down to not lose quality
 482             for density in densities:
 483                 if density not in empty_densities:
 484                     last_density = density
 485                     continue
 486                 if last_density is None:
 487                     continue
 488                 if options.verbose:
 489                     print "Density %s not available, resizing down from %s" % (
 490                             density, last_density)
 491
 492                 last_iconpath = os.path.join(
 493                         get_icon_dir(repodir, last_density), iconfilename)
 494                 iconpath = os.path.join(
 495                         get_icon_dir(repodir, density), iconfilename)
 496                 im = Image.open(last_iconpath)
 497                 size = launcher_size(density)
 498
 499                 im.thumbnail((size, size), Image.ANTIALIAS)
 500                 im.save(iconpath, "PNG")
 501                 empty_densities.remove(density)
 502
 503             last_density = None
 504             # Then just copy from the highest resolution available
 505             for density in reversed(densities):
 506                 if density not in empty_densities:
 507                     last_density = density
 508                     continue
 509                 if last_density is None:
 510                     continue
 511                 if options.verbose:
 512                     print "Density %s not available, copying from lower density %s" % (
 513                             density, last_density)
 514
 515                 shutil.copyfile(
 516                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 517                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 518
 519                 empty_densities.remove(density)
 520
 521             # Copy from icons-mdpi to icons since mdpi is the baseline density
 522             shutil.copyfile(
 523                     os.path.join(get_icon_dir(repodir, '160'), iconfilename),
 524                     os.path.join(get_icon_dir(repodir, None), iconfilename))
 525
 526             # Record in known apks, getting the added date at the same time..
 527             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 528             if added:
 529                 thisinfo['added'] = added
 530
 531             apkcache[apkfilename] = thisinfo
 532             cachechanged = True
 533
 534         apks.append(thisinfo)
 535
 536     return apks, cachechanged
 537
 538
 539 repo_pubkey_fingerprint = None
 540
 541 def make_index(apps, apks, repodir, archive, categories):
 542     """Make a repo index.
 543
 544     :param apps: fully populated apps list
 545     :param apks: full populated apks list
 546     :param repodir: the repo directory
 547     :param archive: True if this is the archive repo, False if it's the
 548                     main one.
 549     :param categories: list of categories
 550     """
 551
 552     doc = Document()
 553
 554     def addElement(name, value, doc, parent):
 555         el = doc.createElement(name)
 556         el.appendChild(doc.createTextNode(value))
 557         parent.appendChild(el)
 558     def addElementCDATA(name, value, doc, parent):
 559         el = doc.createElement(name)
 560         el.appendChild(doc.createCDATASection(value))
 561         parent.appendChild(el)
 562
 563     root = doc.createElement("fdroid")
 564     doc.appendChild(root)
 565
 566     repoel = doc.createElement("repo")
 567
 568     if archive:
 569         repoel.setAttribute("name", config['archive_name'])
 570         if config['repo_maxage'] != 0:
 571             repoel.setAttribute("maxage", str(config['repo_maxage']))
 572         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 573         repoel.setAttribute("url", config['archive_url'])
 574         addElement('description', config['archive_description'], doc, repoel)
 575
 576     else:
 577         repoel.setAttribute("name", config['repo_name'])
 578         if config['repo_maxage'] != 0:
 579             repoel.setAttribute("maxage", str(config['repo_maxage']))
 580         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 581         repoel.setAttribute("url", config['repo_url'])
 582         addElement('description', config['repo_description'], doc, repoel)
 583
 584     repoel.setAttribute("version", "10")
 585     repoel.setAttribute("timestamp", str(int(time.time())))
 586
 587     if config['repo_keyalias']:
 588
 589         # Generate a certificate fingerprint the same way keytool does it
 590         # (but with slightly different formatting)
 591         def cert_fingerprint(data):
 592             digest = hashlib.sha1(data).digest()
 593             ret = []
 594             for i in range(4):
 595                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 596             return " ".join(ret)
 597
 598         def extract_pubkey():
 599             p = subprocess.Popen(['keytool', '-exportcert',
 600                                   '-alias', config['repo_keyalias'],
 601                                   '-keystore', config['keystore'],
 602                                   '-storepass', config['keystorepass']],
 603                                  stdout=subprocess.PIPE)
 604             cert = p.communicate()[0]
 605             if p.returncode != 0:
 606                 print "ERROR: Failed to get repo pubkey"
 607                 sys.exit(1)
 608             global repo_pubkey_fingerprint
 609             repo_pubkey_fingerprint = cert_fingerprint(cert)
 610             return "".join("%02x" % ord(b) for b in cert)
 611
 612         repoel.setAttribute("pubkey", extract_pubkey())
 613
 614     root.appendChild(repoel)
 615
 616     for app in apps:
 617
 618         if app['Disabled'] is not None:
 619             continue
 620
 621         # Get a list of the apks for this app...
 622         apklist = []
 623         for apk in apks:
 624             if apk['id'] == app['id']:
 625                 apklist.append(apk)
 626
 627         if len(apklist) == 0:
 628             continue
 629
 630         apel = doc.createElement("application")
 631         apel.setAttribute("id", app['id'])
 632         root.appendChild(apel)
 633
 634         addElement('id', app['id'], doc, apel)
 635         if 'added' in app:
 636             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 637         if 'lastupdated' in app:
 638             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 639         addElement('name', app['Name'], doc, apel)
 640         addElement('summary', app['Summary'], doc, apel)
 641         if app['icon']:
 642             addElement('icon', app['icon'], doc, apel)
 643         def linkres(link):
 644             for app in apps:
 645                 if app['id'] == link:
 646                     return ("fdroid.app:" + link, app['Name'])
 647             raise MetaDataException("Cannot resolve app id " + link)
 648         addElement('desc',
 649                 metadata.description_html(app['Description'], linkres), doc, apel)
 650         addElement('license', app['License'], doc, apel)
 651         if 'Categories' in app:
 652             appcategories = [c.strip() for c in app['Categories'].split(',')]
 653             addElement('categories', ','.join(appcategories), doc, apel)
 654             # We put the first (primary) category in LAST, which will have
 655             # the desired effect of making clients that only understand one
 656             # category see that one.
 657             addElement('category', appcategories[0], doc, apel)
 658         addElement('web', app['Web Site'], doc, apel)
 659         addElement('source', app['Source Code'], doc, apel)
 660         addElement('tracker', app['Issue Tracker'], doc, apel)
 661         if app['Donate']:
 662             addElement('donate', app['Donate'], doc, apel)
 663         if app['Bitcoin']:
 664             addElement('bitcoin', app['Bitcoin'], doc, apel)
 665         if app['Litecoin']:
 666             addElement('litecoin', app['Litecoin'], doc, apel)
 667         if app['Dogecoin']:
 668             addElement('dogecoin', app['Dogecoin'], doc, apel)
 669         if app['FlattrID']:
 670             addElement('flattr', app['FlattrID'], doc, apel)
 671
 672         # These elements actually refer to the current version (i.e. which
 673         # one is recommended. They are historically mis-named, and need
 674         # changing, but stay like this for now to support existing clients.
 675         addElement('marketversion', app['Current Version'], doc, apel)
 676         addElement('marketvercode', app['Current Version Code'], doc, apel)
 677
 678         if app['AntiFeatures']:
 679             af = app['AntiFeatures'].split(',')
 680             # TODO: Temporarily not including UpstreamNonFree in the index,
 681             # because current F-Droid clients do not understand it, and also
 682             # look ugly when they encounter an unknown antifeature. This
 683             # filtering can be removed in time...
 684             if 'UpstreamNonFree' in af:
 685                 af.remove('UpstreamNonFree')
 686             if af:
 687                 addElement('antifeatures', ','.join(af), doc, apel)
 688         if app['Provides']:
 689             pv = app['Provides'].split(',')
 690             addElement('provides', ','.join(pv), doc, apel)
 691         if app['Requires Root']:
 692             addElement('requirements', 'root', doc, apel)
 693
 694         # Sort the apk list into version order, just so the web site
 695         # doesn't have to do any work by default...
 696         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 697
 698         # Check for duplicates - they will make the client unhappy...
 699         for i in range(len(apklist) - 1):
 700             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 701                 print "ERROR - duplicate versions"
 702                 print apklist[i]['apkname']
 703                 print apklist[i+1]['apkname']
 704                 sys.exit(1)
 705
 706         for apk in apklist:
 707             apkel = doc.createElement("package")
 708             apel.appendChild(apkel)
 709             addElement('version', apk['version'], doc, apkel)
 710             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 711             addElement('apkname', apk['apkname'], doc, apkel)
 712             if 'srcname' in apk:
 713                 addElement('srcname', apk['srcname'], doc, apkel)
 714             for hash_type in ['sha256']:
 715                 if not hash_type in apk:
 716                     continue
 717                 hashel = doc.createElement("hash")
 718                 hashel.setAttribute("type", hash_type)
 719                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 720                 apkel.appendChild(hashel)
 721             addElement('sig', apk['sig'], doc, apkel)
 722             addElement('size', str(apk['size']), doc, apkel)
 723             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 724             if 'added' in apk:
 725                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 726             if app['Requires Root']:
 727                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 728                     apk['permissions'].append('ACCESS_SUPERUSER')
 729
 730             if len(apk['permissions']) > 0:
 731                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 732             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 733                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 734             if len(apk['features']) > 0:
 735                 addElement('features', ','.join(apk['features']), doc, apkel)
 736
 737     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 738     if options.pretty:
 739         output = doc.toprettyxml()
 740     else:
 741         output = doc.toxml()
 742     of.write(output)
 743     of.close()
 744
 745     if config['repo_keyalias'] is not None:
 746
 747         if not options.quiet:
 748             print "Creating signed index."
 749             print "Key fingerprint:", repo_pubkey_fingerprint
 750
 751         #Create a jar of the index...
 752         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 753             cwd=repodir, stdout=subprocess.PIPE)
 754         output = p.communicate()[0]
 755         if options.verbose:
 756             print output
 757         if p.returncode != 0:
 758             print "ERROR: Failed to create jar file"
 759             sys.exit(1)
 760
 761         # Sign the index...
 762         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 763             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 764             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 765             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 766         output = p.communicate()[0]
 767         if p.returncode != 0:
 768             print "Failed to sign index"
 769             print output
 770             sys.exit(1)
 771         if options.verbose:
 772             print output
 773
 774     # Copy the repo icon into the repo directory...
 775     icon_dir = os.path.join(repodir ,'icons')
 776     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 777     shutil.copyfile(config['repo_icon'], iconfilename)
 778
 779     # Write a category list in the repo to allow quick access...
 780     catdata = ''
 781     for cat in categories:
 782         catdata += cat + '\n'
 783     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 784     f.write(catdata)
 785     f.close()
 786
 787
 788
 789 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 790
 791     for app in apps:
 792
 793         # Get a list of the apks for this app...
 794         apklist = []
 795         for apk in apks:
 796             if apk['id'] == app['id']:
 797                 apklist.append(apk)
 798
 799         # Sort the apk list into version order...
 800         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 801
 802         if app['Archive Policy']:
 803             keepversions = int(app['Archive Policy'][:-9])
 804         else:
 805             keepversions = defaultkeepversions
 806
 807         if len(apklist) > keepversions:
 808             for apk in apklist[keepversions:]:
 809                 print "Moving " + apk['apkname'] + " to archive"
 810                 shutil.move(os.path.join(repodir, apk['apkname']),
 811                     os.path.join(archivedir, apk['apkname']))
 812                 if 'srcname' in apk:
 813                     shutil.move(os.path.join(repodir, apk['srcname']),
 814                         os.path.join(archivedir, apk['srcname']))
 815                 apks.remove(apk)
 816
 817
 818 config = None
 819 options = None
 820
 821 def main():
 822
 823     global config, options
 824
 825     # Parse command line...
 826     parser = OptionParser()
 827     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 828                       help="Create skeleton metadata files that are missing")
 829     parser.add_option("-v", "--verbose", action="store_true", default=False,
 830                       help="Spew out even more information than normal")
 831     parser.add_option("-q", "--quiet", action="store_true", default=False,
 832                       help="No output, except for warnings and errors")
 833     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 834                       help="Report on build data status")
 835     parser.add_option("-i", "--interactive", default=False, action="store_true",
 836                       help="Interactively ask about things that need updating.")
 837     parser.add_option("-I", "--icons", action="store_true", default=False,
 838                       help="Resize all the icons exceeding the max pixel size and exit")
 839     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 840                       help="Specify editor to use in interactive mode. Default "+
 841                           "is /etc/alternatives/editor")
 842     parser.add_option("-w", "--wiki", default=False, action="store_true",
 843                       help="Update the wiki")
 844     parser.add_option("", "--pretty", action="store_true", default=False,
 845                       help="Produce human-readable index.xml")
 846     parser.add_option("--clean", action="store_true", default=False,
 847                       help="Clean update - don't uses caches, reprocess all apks")
 848     (options, args) = parser.parse_args()
 849
 850     config = common.read_config(options)
 851
 852     repodirs = ['repo']
 853     if config['archive_older'] != 0:
 854         repodirs.append('archive')
 855         if not os.path.exists('archive'):
 856             os.mkdir('archive')
 857
 858     if options.icons:
 859         resize_all_icons(repodirs)
 860         sys.exit(0)
 861
 862     # Get all apps...
 863     apps = metadata.read_metadata()
 864
 865     # Generate a list of categories...
 866     categories = []
 867     for app in apps:
 868         cats = app['Categories'].split(',')
 869         for cat in cats:
 870             if cat not in categories:
 871                 categories.append(cat)
 872
 873     # Read known apks data (will be updated and written back when we've finished)
 874     knownapks = common.KnownApks()
 875
 876     # Gather information about all the apk files in the repo directory, using
 877     # cached data if possible.
 878     apkcachefile = os.path.join('tmp', 'apkcache')
 879     if not options.clean and os.path.exists(apkcachefile):
 880         with open(apkcachefile, 'rb') as cf:
 881             apkcache = pickle.load(cf)
 882     else:
 883         apkcache = {}
 884     cachechanged = False
 885
 886     delete_disabled_builds(apps, apkcache, repodirs)
 887
 888     # Scan all apks in the main repo
 889     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 890     if cc:
 891         cachechanged = True
 892
 893     # Scan the archive repo for apks as well
 894     if len(repodirs) > 1:
 895         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 896         if cc:
 897             cachechanged = True
 898     else:
 899         archapks = []
 900
 901     # Some information from the apks needs to be applied up to the application
 902     # level. When doing this, we use the info from the most recent version's apk.
 903     # We deal with figuring out when the app was added and last updated at the
 904     # same time.
 905     for app in apps:
 906         bestver = 0
 907         added = None
 908         lastupdated = None
 909         for apk in apks + archapks:
 910             if apk['id'] == app['id']:
 911                 if apk['versioncode'] > bestver:
 912                     bestver = apk['versioncode']
 913                     bestapk = apk
 914
 915                 if 'added' in apk:
 916                     if not added or apk['added'] < added:
 917                         added = apk['added']
 918                     if not lastupdated or apk['added'] > lastupdated:
 919                         lastupdated = apk['added']
 920
 921         if added:
 922             app['added'] = added
 923         else:
 924             if options.verbose:
 925                 print "WARNING: Don't know when " + app['id'] + " was added"
 926         if lastupdated:
 927             app['lastupdated'] = lastupdated
 928         else:
 929             if options.verbose:
 930                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 931
 932         if bestver == 0:
 933             if app['Name'] is None:
 934                 app['Name'] = app['id']
 935             app['icon'] = None
 936             if options.verbose and app['Disabled'] is None:
 937                 print "WARNING: Application " + app['id'] + " has no packages"
 938         else:
 939             if app['Name'] is None:
 940                 app['Name'] = bestapk['name']
 941             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 942
 943     # Sort the app list by name, then the web site doesn't have to by default.
 944     # (we had to wait until we'd scanned the apks to do this, because mostly the
 945     # name comes from there!)
 946     apps = sorted(apps, key=lambda app: app['Name'].upper())
 947
 948     # Generate warnings for apk's with no metadata (or create skeleton
 949     # metadata files, if requested on the command line)
 950     for apk in apks:
 951         found = False
 952         for app in apps:
 953             if app['id'] == apk['id']:
 954                 found = True
 955                 break
 956         if not found:
 957             if options.createmeta:
 958                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 959                 f.write("License:Unknown\n")
 960                 f.write("Web Site:\n")
 961                 f.write("Source Code:\n")
 962                 f.write("Issue Tracker:\n")
 963                 f.write("Summary:" + apk['name'] + "\n")
 964                 f.write("Description:\n")
 965                 f.write(apk['name'] + "\n")
 966                 f.write(".\n")
 967                 f.close()
 968                 print "Generated skeleton metadata for " + apk['id']
 969             else:
 970                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 971                 print "       " + apk['name'] + " - " + apk['version']
 972
 973     if len(repodirs) > 1:
 974         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 975
 976     # Make the index for the main repo...
 977     make_index(apps, apks, repodirs[0], False, categories)
 978
 979     # If there's an archive repo,  make the index for it. We already scanned it
 980     # earlier on.
 981     if len(repodirs) > 1:
 982         make_index(apps, archapks, repodirs[1], True, categories)
 983
 984     if config['update_stats']:
 985
 986         # Update known apks info...
 987         knownapks.writeifchanged()
 988
 989         # Generate latest apps data for widget
 990         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 991             data = ''
 992             for line in file(os.path.join('stats', 'latestapps.txt')):
 993                 appid = line.rstrip()
 994                 data += appid + "\t"
 995                 for app in apps:
 996                     if app['id'] == appid:
 997                         data += app['Name'] + "\t"
 998                         if app['icon'] is not None:
 999                             data += app['icon'] + "\t"
1000                         data += app['License'] + "\n"
1001                         break
1002             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1003             f.write(data)
1004             f.close()
1005
1006     if cachechanged:
1007         with open(apkcachefile, 'wb') as cf:
1008             pickle.dump(apkcache, cf)
1009
1010     # Update the wiki...
1011     if options.wiki:
1012         update_wiki(apps, apks + archapks)
1013
1014     print "Finished."
1015
1016 if __name__ == "__main__":
1017     main()
1018