fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import subprocess
  26 import re
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 import common, metadata
  34 from metadata import MetaDataException
  35 from PIL import Image
  36
  37 def update_wiki(apps, apks):
  38     """Update the wiki
  39
  40     :param apps: fully populated list of all applications
  41     :param apks: all apks, except...
  42     """
  43     print "Updating wiki"
  44     wikicat = 'Apps'
  45     wikiredircat = 'App Redirects'
  46     import mwclient
  47     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  48             path=config['wiki_path'])
  49     site.login(config['wiki_user'], config['wiki_password'])
  50     generated_pages = {}
  51     generated_redirects = {}
  52     for app in apps:
  53         wikidata = ''
  54         if app['Disabled']:
  55             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  56         if app['AntiFeatures']:
  57             for af in app['AntiFeatures'].split(','):
  58                 wikidata += '{{AntiFeature|' + af + '}}\n'
  59         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  60                 app['id'],
  61                 app['Name'],
  62                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  63                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  64                 app['Source Code'],
  65                 app['Issue Tracker'],
  66                 app['Web Site'],
  67                 app['Donate'],
  68                 app['FlattrID'],
  69                 app['Bitcoin'],
  70                 app['Litecoin'],
  71                 app['Dogecoin'],
  72                 app['License'],
  73                 app.get('Requires Root', 'No'))
  74
  75         if app['Provides']:
  76             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  77
  78         wikidata += app['Summary']
  79         wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
  80
  81         wikidata += "=Description=\n"
  82         wikidata += metadata.description_wiki(app['Description']) + "\n"
  83
  84         wikidata += "=Maintainer Notes=\n"
  85         if 'Maintainer Notes' in app:
  86             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
  87         wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
  88
  89         # Get a list of all packages for this application...
  90         apklist = []
  91         gotcurrentver = False
  92         cantupdate = False
  93         buildfails = False
  94         for apk in apks:
  95             if apk['id'] == app['id']:
  96                 if str(apk['versioncode']) == app['Current Version Code']:
  97                     gotcurrentver = True
  98                 apklist.append(apk)
  99         # Include ones we can't build, as a special case...
 100         for thisbuild in app['builds']:
 101             if 'disable' in thisbuild:
 102                 if thisbuild['vercode'] == app['Current Version Code']:
 103                     cantupdate = True
 104                 apklist.append({
 105                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 106                         'versioncode': int(thisbuild['vercode']),
 107                         'version': thisbuild['version'],
 108                         'buildproblem': thisbuild['disable']
 109                     })
 110             else:
 111                 builtit = False
 112                 for apk in apklist:
 113                     if apk['versioncode'] == int(thisbuild['vercode']):
 114                         builtit = True
 115                         break
 116                 if not builtit:
 117                     buildfails = True
 118                     apklist.append({
 119                             'versioncode': int(thisbuild['vercode']),
 120                             'version': thisbuild['version'],
 121                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 122                         })
 123         # Sort with most recent first...
 124         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 125
 126         wikidata += "=Versions=\n"
 127         if len(apklist) == 0:
 128             wikidata += "We currently have no versions of this app available."
 129         elif not gotcurrentver:
 130             wikidata += "We don't have the current version of this app."
 131         else:
 132             wikidata += "We have the current version of this app."
 133         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 134         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 135         if len(app['No Source Since']) > 0:
 136             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 137         if len(app['Current Version']) > 0:
 138             wikidata += "The current (recommended) version is " + app['Current Version']
 139             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 140         validapks = 0
 141         for apk in apklist:
 142             wikidata += "==" + apk['version'] + "==\n"
 143
 144             if 'buildproblem' in apk:
 145                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 146             else:
 147                 validapks += 1
 148                 wikidata += "This version is built and signed by "
 149                 if 'srcname' in apk:
 150                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 151                 else:
 152                     wikidata += "the original developer.\n\n"
 153             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 154
 155         wikidata += '\n[[Category:' + wikicat + ']]\n'
 156         if len(app['No Source Since']) > 0:
 157             wikidata += '\n[[Category:Apps missing source code]]\n'
 158         elif validapks == 0 and not app['Disabled']:
 159             wikidata += '\n[[Category:Apps with no packages]]\n'
 160         elif cantupdate and not app['Disabled']:
 161             wikidata += "\n[[Category:Apps we can't update]]\n"
 162         elif buildfails and not app['Disabled']:
 163             wikidata += "\n[[Category:Apps with failing builds]]\n"
 164         elif not gotcurrentver and not app['Disabled'] and app['Update Check Mode'] != "Static":
 165             wikidata += '\n[[Category:Apps to Update]]\n'
 166         if app['Disabled']:
 167             wikidata += '\n[[Category:Apps that are disabled]]\n'
 168         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 169             wikidata += '\n[[Category:Apps with no update check]]\n'
 170
 171         # We can't have underscores in the page name, even if they're in
 172         # the package ID, because MediaWiki messes with them...
 173         pagename = app['id'].replace('_', ' ')
 174
 175         # Drop a trailing newline, because mediawiki is going to drop it anyway
 176         # and it we don't we'll think the page has changed when it hasn't...
 177         if wikidata.endswith('\n'):
 178             wikidata = wikidata[:-1]
 179
 180         generated_pages[pagename] = wikidata
 181
 182         # Make a redirect from the name to the ID too, unless there's
 183         # already an existing page with the name and it isn't a redirect.
 184         noclobber = False
 185         apppagename = app['Name'].replace('_', ' ')
 186         apppagename = apppagename.replace('{', '')
 187         apppagename = apppagename.replace('}', ' ')
 188         apppagename = apppagename.replace(':', ' ')
 189         # Drop double spaces caused mostly by replacing ':' above
 190         apppagename = apppagename.replace('  ', ' ')
 191         for expagename in site.allpages(prefix=apppagename,
 192                 filterredir='nonredirects', generator=False):
 193             if expagename == apppagename:
 194                 noclobber = True
 195         # Another reason not to make the redirect page is if the app name
 196         # is the same as it's ID, because that will overwrite the real page
 197         # with an redirect to itself! (Although it seems like an odd
 198         # scenario this happens a lot, e.g. where there is metadata but no
 199         # builds or binaries to extract a name from.
 200         if apppagename == pagename:
 201             noclobber = True
 202         if not noclobber:
 203             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 204
 205     for tcat, genp in [(wikicat, generated_pages),
 206             (wikiredircat, generated_redirects)]:
 207         catpages = site.Pages['Category:' + tcat]
 208         existingpages = []
 209         for page in catpages:
 210             existingpages.append(page.name)
 211             if page.name in genp:
 212                 pagetxt = page.edit()
 213                 if pagetxt != genp[page.name]:
 214                     print "Updating modified page " + page.name
 215                     page.save(genp[page.name], summary='Auto-updated')
 216                 else:
 217                     if options.verbose:
 218                         print "Page " + page.name + " is unchanged"
 219             else:
 220                 print "Deleting page " + page.name
 221                 page.delete('No longer published')
 222         for pagename, text in genp.items():
 223             if options.verbose:
 224                 print "Checking " + pagename
 225             if not pagename in existingpages:
 226                 print "Creating page " + pagename
 227                 try:
 228                     newpage = site.Pages[pagename]
 229                     newpage.save(text, summary='Auto-created')
 230                 except:
 231                     print "...FAILED to create page"
 232
 233     # Purge server cache to ensure counts are up to date
 234     site.pages['Repository Maintenance'].purge()
 235
 236 def delete_disabled_builds(apps, apkcache, repodirs):
 237     """Delete disabled build outputs.
 238
 239     :param apps: list of all applications, as per metadata.read_metadata
 240     :param apkcache: current apk cache information
 241     :param repodirs: the repo directories to process
 242     """
 243     for app in apps:
 244         for build in app['builds']:
 245             if 'disable' in build:
 246                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 247                 for repodir in repodirs:
 248                     apkpath = os.path.join(repodir, apkfilename)
 249                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 250                     for name in [apkpath, srcpath]:
 251                         if os.path.exists(name):
 252                             print "Deleting disabled build output " + apkfilename
 253                             os.remove(name)
 254                 if apkfilename in apkcache:
 255                     del apkcache[apkfilename]
 256
 257 def resize_icon(iconpath):
 258     try:
 259         im = Image.open(iconpath)
 260         if any(length > config['max_icon_size'] for length in im.size):
 261             print iconpath, "is too large:", im.size
 262             im.thumbnail((config['max_icon_size'], config['max_icon_size']),
 263                     Image.ANTIALIAS)
 264             print iconpath, "new size:", im.size
 265             im.save(iconpath, "PNG")
 266         else:
 267             if options.verbose:
 268                 print iconpath, "is small enough:", im.size
 269     except Exception,e:
 270         print "ERROR: Failed processing {0} - {1}".format(iconpath, e)
 271
 272 def resize_all_icons(repodirs):
 273     """Resize all icons that exceed the max size
 274
 275     :param apps: list of all applications, as per metadata.read_metadata
 276     :param repodirs: the repo directories to process
 277     """
 278     for repodir in repodirs:
 279         for iconpath in glob.glob(os.path.join(repodir, 'icons', '*.png')):
 280             resize_icon(iconpath)
 281
 282 def scan_apks(apps, apkcache, repodir, knownapks):
 283     """Scan the apks in the given repo directory.
 284
 285     This also extracts the icons.
 286
 287     :param apps: list of all applications, as per metadata.read_metadata
 288     :param apkcache: current apk cache information
 289     :param repodir: repo directory to scan
 290     :param knownapks: known apks info
 291     :returns: (apks, cachechanged) where apks is a list of apk information,
 292               and cachechanged is True if the apkcache got changed.
 293     """
 294
 295     cachechanged = False
 296
 297     icon_dir = os.path.join(repodir ,'icons')
 298     # Delete and re-create the icon directory...
 299     if options.clean and os.path.exists(icon_dir):
 300         shutil.rmtree(icon_dir)
 301     if not os.path.exists(icon_dir):
 302         os.makedirs(icon_dir)
 303     apks = []
 304     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 305     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 306     vername_pat = re.compile(".*versionName='([^']*)'.*")
 307     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 308     icon_pat = re.compile(".*icon='([^']+?)'.*")
 309     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 310     string_pat = re.compile(".*'([^']*)'.*")
 311     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 312
 313         apkfilename = apkfile[len(repodir) + 1:]
 314         if ' ' in apkfilename:
 315             print "No spaces in APK filenames!"
 316             sys.exit(1)
 317
 318         if apkfilename in apkcache:
 319             if options.verbose:
 320                 print "Reading " + apkfilename + " from cache"
 321             thisinfo = apkcache[apkfilename]
 322
 323         else:
 324
 325             if not options.quiet:
 326                 print "Processing " + apkfilename
 327             thisinfo = {}
 328             thisinfo['apkname'] = apkfilename
 329             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 330             if os.path.exists(os.path.join(repodir, srcfilename)):
 331                 thisinfo['srcname'] = srcfilename
 332             thisinfo['size'] = os.path.getsize(apkfile)
 333             thisinfo['permissions'] = []
 334             thisinfo['features'] = []
 335             p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
 336                                   'dump', 'badging', apkfile],
 337                                  stdout=subprocess.PIPE)
 338             output = p.communicate()[0]
 339             if options.verbose:
 340                 print output
 341             if p.returncode != 0:
 342                 print "ERROR: Failed to get apk information"
 343                 sys.exit(1)
 344             for line in output.splitlines():
 345                 if line.startswith("package:"):
 346                     try:
 347                         thisinfo['id'] = re.match(name_pat, line).group(1)
 348                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 349                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 350                     except Exception, e:
 351                         print "Package matching failed: " + str(e)
 352                         print "Line was: " + line
 353                         sys.exit(1)
 354                 elif line.startswith("application:"):
 355                     thisinfo['name'] = re.match(label_pat, line).group(1)
 356                     match = re.match(icon_pat, line)
 357                     if match:
 358                         thisinfo['iconsrc'] = match.group(1)
 359                 elif line.startswith("sdkVersion:"):
 360                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 361                 elif line.startswith("native-code:"):
 362                     thisinfo['nativecode'] = []
 363                     for arch in line[13:].split(' '):
 364                         thisinfo['nativecode'].append(arch[1:-1])
 365                 elif line.startswith("uses-permission:"):
 366                     perm = re.match(string_pat, line).group(1)
 367                     if perm.startswith("android.permission."):
 368                         perm = perm[19:]
 369                     thisinfo['permissions'].append(perm)
 370                 elif line.startswith("uses-feature:"):
 371                     perm = re.match(string_pat, line).group(1)
 372                     #Filter out this, it's only added with the latest SDK tools and
 373                     #causes problems for lots of apps.
 374                     if (perm != "android.hardware.screen.portrait" and
 375                         perm != "android.hardware.screen.landscape"):
 376                         if perm.startswith("android.feature."):
 377                             perm = perm[16:]
 378                         thisinfo['features'].append(perm)
 379
 380             if not 'sdkversion' in thisinfo:
 381                 print "  WARNING: no SDK version information found"
 382                 thisinfo['sdkversion'] = 0
 383
 384             # Check for debuggable apks...
 385             if common.isApkDebuggable(apkfile, config):
 386                 print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
 387
 388             # Calculate the sha256...
 389             sha = hashlib.sha256()
 390             with open(apkfile, 'rb') as f:
 391                 while True:
 392                     t = f.read(1024)
 393                     if len(t) == 0:
 394                         break
 395                     sha.update(t)
 396                 thisinfo['sha256'] = sha.hexdigest()
 397
 398             # Get the signature (or md5 of, to be precise)...
 399             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 400             if not os.path.exists(getsig_dir + "/getsig.class"):
 401                 print "ERROR: getsig.class not found. To fix:"
 402                 print "\tcd " + getsig_dir
 403                 print "\t./make.sh"
 404                 sys.exit(1)
 405             p = subprocess.Popen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 406                         'getsig', os.path.join(os.getcwd(), apkfile)], stdout=subprocess.PIPE)
 407             output = p.communicate()[0]
 408             if options.verbose:
 409                 print output
 410             if p.returncode != 0 or not output.startswith('Result:'):
 411                 print "ERROR: Failed to get apk signature"
 412                 sys.exit(1)
 413             thisinfo['sig'] = output[7:].strip()
 414
 415             # Extract the icon file...
 416             if 'iconsrc' in thisinfo:
 417                 apk = zipfile.ZipFile(apkfile, 'r')
 418                 thisinfo['icon'] = (thisinfo['id'] + '.' +
 419                     str(thisinfo['versioncode']) + '.png')
 420                 iconpath = os.path.join(icon_dir, thisinfo['icon'])
 421                 try:
 422                     iconfile = open(iconpath, 'wb')
 423                     iconfile.write(apk.read(thisinfo['iconsrc']))
 424                     iconfile.close()
 425                 except:
 426                     print "WARNING: Error retrieving icon file"
 427                 apk.close()
 428
 429                 resize_icon(iconpath)
 430
 431             # Record in known apks, getting the added date at the same time..
 432             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 433             if added:
 434                 thisinfo['added'] = added
 435
 436             apkcache[apkfilename] = thisinfo
 437             cachechanged = True
 438
 439         apks.append(thisinfo)
 440
 441     return apks, cachechanged
 442
 443
 444 repo_pubkey_fingerprint = None
 445
 446 def make_index(apps, apks, repodir, archive, categories):
 447     """Make a repo index.
 448
 449     :param apps: fully populated apps list
 450     :param apks: full populated apks list
 451     :param repodir: the repo directory
 452     :param archive: True if this is the archive repo, False if it's the
 453                     main one.
 454     :param categories: list of categories
 455     """
 456
 457     doc = Document()
 458
 459     def addElement(name, value, doc, parent):
 460         el = doc.createElement(name)
 461         el.appendChild(doc.createTextNode(value))
 462         parent.appendChild(el)
 463     def addElementCDATA(name, value, doc, parent):
 464         el = doc.createElement(name)
 465         el.appendChild(doc.createCDATASection(value))
 466         parent.appendChild(el)
 467
 468     root = doc.createElement("fdroid")
 469     doc.appendChild(root)
 470
 471     repoel = doc.createElement("repo")
 472     if archive:
 473         repoel.setAttribute("name", config['archive_name'])
 474         if config['repo_maxage'] != 0:
 475             repoel.setAttribute("maxage", str(config['repo_maxage']))
 476         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 477         repoel.setAttribute("url", config['archive_url'])
 478         addElement('description', config['archive_description'], doc, repoel)
 479     else:
 480         repoel.setAttribute("name", config['repo_name'])
 481         if config['repo_maxage'] != 0:
 482             repoel.setAttribute("maxage", str(config['repo_maxage']))
 483         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 484         repoel.setAttribute("url", config['repo_url'])
 485         addElement('description', config['repo_description'], doc, repoel)
 486     repoel.setAttribute("timestamp", str(int(time.time())))
 487
 488     if config['repo_keyalias']:
 489
 490         # Generate a certificate fingerprint the same way keytool does it
 491         # (but with slightly different formatting)
 492         def cert_fingerprint(data):
 493             digest = hashlib.sha1(data).digest()
 494             ret = []
 495             for i in range(4):
 496                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 497             return " ".join(ret)
 498
 499         def extract_pubkey():
 500             p = subprocess.Popen(['keytool', '-exportcert',
 501                                   '-alias', config['repo_keyalias'],
 502                                   '-keystore', config['keystore'],
 503                                   '-storepass', config['keystorepass']],
 504                                  stdout=subprocess.PIPE)
 505             cert = p.communicate()[0]
 506             if p.returncode != 0:
 507                 print "ERROR: Failed to get repo pubkey"
 508                 sys.exit(1)
 509             global repo_pubkey_fingerprint
 510             repo_pubkey_fingerprint = cert_fingerprint(cert)
 511             return "".join("%02x" % ord(b) for b in cert)
 512
 513         repoel.setAttribute("pubkey", extract_pubkey())
 514
 515     root.appendChild(repoel)
 516
 517     for app in apps:
 518
 519         if app['Disabled'] is not None:
 520             continue
 521
 522         # Get a list of the apks for this app...
 523         apklist = []
 524         for apk in apks:
 525             if apk['id'] == app['id']:
 526                 apklist.append(apk)
 527
 528         if len(apklist) == 0:
 529             continue
 530
 531         apel = doc.createElement("application")
 532         apel.setAttribute("id", app['id'])
 533         root.appendChild(apel)
 534
 535         addElement('id', app['id'], doc, apel)
 536         if 'added' in app:
 537             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 538         if 'lastupdated' in app:
 539             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 540         addElement('name', app['Name'], doc, apel)
 541         addElement('summary', app['Summary'], doc, apel)
 542         if app['icon']:
 543             addElement('icon', app['icon'], doc, apel)
 544         def linkres(link):
 545             for app in apps:
 546                 if app['id'] == link:
 547                     return ("fdroid.app:" + link, app['Name'])
 548             raise MetaDataException("Cannot resolve app id " + link)
 549         addElement('desc',
 550                 metadata.description_html(app['Description'], linkres), doc, apel)
 551         addElement('license', app['License'], doc, apel)
 552         if 'Categories' in app:
 553             appcategories = [c.strip() for c in app['Categories'].split(',')]
 554             addElement('categories', ','.join(appcategories), doc, apel)
 555             # We put the first (primary) category in LAST, which will have
 556             # the desired effect of making clients that only understand one
 557             # category see that one.
 558             addElement('category', appcategories[0], doc, apel)
 559         addElement('web', app['Web Site'], doc, apel)
 560         addElement('source', app['Source Code'], doc, apel)
 561         addElement('tracker', app['Issue Tracker'], doc, apel)
 562         if app['Donate']:
 563             addElement('donate', app['Donate'], doc, apel)
 564         if app['Bitcoin']:
 565             addElement('bitcoin', app['Bitcoin'], doc, apel)
 566         if app['Litecoin']:
 567             addElement('litecoin', app['Litecoin'], doc, apel)
 568         if app['Dogecoin']:
 569             addElement('dogecoin', app['Dogecoin'], doc, apel)
 570         if app['FlattrID']:
 571             addElement('flattr', app['FlattrID'], doc, apel)
 572
 573         # These elements actually refer to the current version (i.e. which
 574         # one is recommended. They are historically mis-named, and need
 575         # changing, but stay like this for now to support existing clients.
 576         addElement('marketversion', app['Current Version'], doc, apel)
 577         addElement('marketvercode', app['Current Version Code'], doc, apel)
 578
 579         if app['AntiFeatures']:
 580             af = app['AntiFeatures'].split(',')
 581             # TODO: Temporarily not including UpstreamNonFree in the index,
 582             # because current F-Droid clients do not understand it, and also
 583             # look ugly when they encounter an unknown antifeature. This
 584             # filtering can be removed in time...
 585             if 'UpstreamNonFree' in af:
 586                 af.remove('UpstreamNonFree')
 587             if af:
 588                 addElement('antifeatures', ','.join(af), doc, apel)
 589         if app['Provides']:
 590             pv = app['Provides'].split(',')
 591             addElement('provides', ','.join(pv), doc, apel)
 592         if app['Requires Root']:
 593             addElement('requirements', 'root', doc, apel)
 594
 595         # Sort the apk list into version order, just so the web site
 596         # doesn't have to do any work by default...
 597         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 598
 599         # Check for duplicates - they will make the client unhappy...
 600         for i in range(len(apklist) - 1):
 601             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 602                 print "ERROR - duplicate versions"
 603                 print apklist[i]['apkname']
 604                 print apklist[i+1]['apkname']
 605                 sys.exit(1)
 606
 607         for apk in apklist:
 608             apkel = doc.createElement("package")
 609             apel.appendChild(apkel)
 610             addElement('version', apk['version'], doc, apkel)
 611             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 612             addElement('apkname', apk['apkname'], doc, apkel)
 613             if 'srcname' in apk:
 614                 addElement('srcname', apk['srcname'], doc, apkel)
 615             for hash_type in ['sha256']:
 616                 if not hash_type in apk:
 617                     continue
 618                 hashel = doc.createElement("hash")
 619                 hashel.setAttribute("type", hash_type)
 620                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 621                 apkel.appendChild(hashel)
 622             addElement('sig', apk['sig'], doc, apkel)
 623             addElement('size', str(apk['size']), doc, apkel)
 624             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 625             if 'added' in apk:
 626                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 627             if app['Requires Root']:
 628                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 629                     apk['permissions'].append('ACCESS_SUPERUSER')
 630
 631             if len(apk['permissions']) > 0:
 632                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 633             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 634                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 635             if len(apk['features']) > 0:
 636                 addElement('features', ','.join(apk['features']), doc, apkel)
 637
 638     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 639     if options.pretty:
 640         output = doc.toprettyxml()
 641     else:
 642         output = doc.toxml()
 643     of.write(output)
 644     of.close()
 645
 646     if config['repo_keyalias'] is not None:
 647
 648         if not options.quiet:
 649             print "Creating signed index."
 650             print "Key fingerprint:", repo_pubkey_fingerprint
 651
 652         #Create a jar of the index...
 653         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 654             cwd=repodir, stdout=subprocess.PIPE)
 655         output = p.communicate()[0]
 656         if options.verbose:
 657             print output
 658         if p.returncode != 0:
 659             print "ERROR: Failed to create jar file"
 660             sys.exit(1)
 661
 662         # Sign the index...
 663         p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
 664             '-storepass', config['keystorepass'], '-keypass', config['keypass'],
 665             '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 666             os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
 667         output = p.communicate()[0]
 668         if p.returncode != 0:
 669             print "Failed to sign index"
 670             print output
 671             sys.exit(1)
 672         if options.verbose:
 673             print output
 674
 675     # Copy the repo icon into the repo directory...
 676     icon_dir=os.path.join(repodir ,'icons')
 677     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 678     shutil.copyfile(config['repo_icon'], iconfilename)
 679
 680     # Write a category list in the repo to allow quick access...
 681     catdata = ''
 682     for cat in categories:
 683         catdata += cat + '\n'
 684     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 685     f.write(catdata)
 686     f.close()
 687
 688
 689
 690 def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
 691
 692     for app in apps:
 693
 694         # Get a list of the apks for this app...
 695         apklist = []
 696         for apk in apks:
 697             if apk['id'] == app['id']:
 698                 apklist.append(apk)
 699
 700         # Sort the apk list into version order...
 701         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 702
 703         if app['Archive Policy']:
 704             keepversions = int(app['Archive Policy'][:-9])
 705         else:
 706             keepversions = defaultkeepversions
 707
 708         if len(apklist) > keepversions:
 709             for apk in apklist[keepversions:]:
 710                 print "Moving " + apk['apkname'] + " to archive"
 711                 shutil.move(os.path.join(repodir, apk['apkname']),
 712                     os.path.join(archivedir, apk['apkname']))
 713                 if 'srcname' in apk:
 714                     shutil.move(os.path.join(repodir, apk['srcname']),
 715                         os.path.join(archivedir, apk['srcname']))
 716                 apks.remove(apk)
 717
 718
 719 config = None
 720 options = None
 721
 722 def main():
 723
 724     global config, options
 725
 726     # Parse command line...
 727     parser = OptionParser()
 728     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 729                       help="Create skeleton metadata files that are missing")
 730     parser.add_option("-v", "--verbose", action="store_true", default=False,
 731                       help="Spew out even more information than normal")
 732     parser.add_option("-q", "--quiet", action="store_true", default=False,
 733                       help="No output, except for warnings and errors")
 734     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 735                       help="Report on build data status")
 736     parser.add_option("-i", "--interactive", default=False, action="store_true",
 737                       help="Interactively ask about things that need updating.")
 738     parser.add_option("-I", "--icons", action="store_true", default=False,
 739                       help="Resize all the icons exceeding the max pixel size and exit")
 740     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 741                       help="Specify editor to use in interactive mode. Default "+
 742                           "is /etc/alternatives/editor")
 743     parser.add_option("-w", "--wiki", default=False, action="store_true",
 744                       help="Update the wiki")
 745     parser.add_option("", "--pretty", action="store_true", default=False,
 746                       help="Produce human-readable index.xml")
 747     parser.add_option("--clean", action="store_true", default=False,
 748                       help="Clean update - don't uses caches, reprocess all apks")
 749     (options, args) = parser.parse_args()
 750
 751     config = common.read_config(options)
 752
 753     repodirs = ['repo']
 754     if config['archive_older'] != 0:
 755         repodirs.append('archive')
 756         if not os.path.exists('archive'):
 757             os.mkdir('archive')
 758
 759     if options.icons:
 760         resize_all_icons(repodirs)
 761         sys.exit(0)
 762
 763     # Get all apps...
 764     apps = metadata.read_metadata()
 765
 766     # Generate a list of categories...
 767     categories = []
 768     for app in apps:
 769         cats = app['Categories'].split(',')
 770         for cat in cats:
 771             if cat not in categories:
 772                 categories.append(cat)
 773
 774     # Read known apks data (will be updated and written back when we've finished)
 775     knownapks = common.KnownApks()
 776
 777     # Gather information about all the apk files in the repo directory, using
 778     # cached data if possible.
 779     apkcachefile = os.path.join('tmp', 'apkcache')
 780     if not options.clean and os.path.exists(apkcachefile):
 781         with open(apkcachefile, 'rb') as cf:
 782             apkcache = pickle.load(cf)
 783     else:
 784         apkcache = {}
 785     cachechanged = False
 786
 787     delete_disabled_builds(apps, apkcache, repodirs)
 788
 789     # Scan all apks in the main repo
 790     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 791     if cc:
 792         cachechanged = True
 793
 794     # Scan the archive repo for apks as well
 795     if len(repodirs) > 1:
 796         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 797         if cc:
 798             cachechanged = True
 799     else:
 800         archapks = []
 801
 802     # Some information from the apks needs to be applied up to the application
 803     # level. When doing this, we use the info from the most recent version's apk.
 804     # We deal with figuring out when the app was added and last updated at the
 805     # same time.
 806     for app in apps:
 807         bestver = 0
 808         added = None
 809         lastupdated = None
 810         for apk in apks + archapks:
 811             if apk['id'] == app['id']:
 812                 if apk['versioncode'] > bestver:
 813                     bestver = apk['versioncode']
 814                     bestapk = apk
 815
 816                 if 'added' in apk:
 817                     if not added or apk['added'] < added:
 818                         added = apk['added']
 819                     if not lastupdated or apk['added'] > lastupdated:
 820                         lastupdated = apk['added']
 821
 822         if added:
 823             app['added'] = added
 824         else:
 825             if options.verbose:
 826                 print "WARNING: Don't know when " + app['id'] + " was added"
 827         if lastupdated:
 828             app['lastupdated'] = lastupdated
 829         else:
 830             if options.verbose:
 831                 print "WARNING: Don't know when " + app['id'] + " was last updated"
 832
 833         if bestver == 0:
 834             if app['Name'] is None:
 835                 app['Name'] = app['id']
 836             app['icon'] = None
 837             if options.verbose and app['Disabled'] is None:
 838                 print "WARNING: Application " + app['id'] + " has no packages"
 839         else:
 840             if app['Name'] is None:
 841                 app['Name'] = bestapk['name']
 842             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 843
 844     # Sort the app list by name, then the web site doesn't have to by default.
 845     # (we had to wait until we'd scanned the apks to do this, because mostly the
 846     # name comes from there!)
 847     apps = sorted(apps, key=lambda app: app['Name'].upper())
 848
 849     # Generate warnings for apk's with no metadata (or create skeleton
 850     # metadata files, if requested on the command line)
 851     for apk in apks:
 852         found = False
 853         for app in apps:
 854             if app['id'] == apk['id']:
 855                 found = True
 856                 break
 857         if not found:
 858             if options.createmeta:
 859                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 860                 f.write("License:Unknown\n")
 861                 f.write("Web Site:\n")
 862                 f.write("Source Code:\n")
 863                 f.write("Issue Tracker:\n")
 864                 f.write("Summary:" + apk['name'] + "\n")
 865                 f.write("Description:\n")
 866                 f.write(apk['name'] + "\n")
 867                 f.write(".\n")
 868                 f.close()
 869                 print "Generated skeleton metadata for " + apk['id']
 870             else:
 871                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 872                 print "       " + apk['name'] + " - " + apk['version']
 873
 874     if len(repodirs) > 1:
 875         archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
 876
 877     # Make the index for the main repo...
 878     make_index(apps, apks, repodirs[0], False, categories)
 879
 880     # If there's an archive repo,  make the index for it. We already scanned it
 881     # earlier on.
 882     if len(repodirs) > 1:
 883         make_index(apps, archapks, repodirs[1], True, categories)
 884
 885     if config['update_stats']:
 886
 887         # Update known apks info...
 888         knownapks.writeifchanged()
 889
 890         # Generate latest apps data for widget
 891         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
 892             data = ''
 893             for line in file(os.path.join('stats', 'latestapps.txt')):
 894                 appid = line.rstrip()
 895                 data += appid + "\t"
 896                 for app in apps:
 897                     if app['id'] == appid:
 898                         data += app['Name'] + "\t"
 899                         if app['icon'] is not None:
 900                             data += app['icon'] + "\t"
 901                         data += app['License'] + "\n"
 902                         break
 903             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
 904             f.write(data)
 905             f.close()
 906
 907     if cachechanged:
 908         with open(apkcachefile, 'wb') as cf:
 909             pickle.dump(apkcache, cf)
 910
 911     # Update the wiki...
 912     if options.wiki:
 913         update_wiki(apps, apks + archapks)
 914
 915     print "Finished."
 916
 917 if __name__ == "__main__":
 918     main()
 919