fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if 'disable' in thisbuild:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if not pagename in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if 'disable' in build:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304         else:
 305             logging.info("%s is small enough: %s" % im.size)
 306
 307     except Exception, e:
 308         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 309
 310
 311 def resize_all_icons(repodirs):
 312     """Resize all icons that exceed the max size
 313
 314     :param repodirs: the repo directories to process
 315     """
 316     for repodir in repodirs:
 317         for density in get_densities():
 318             icon_dir = get_icon_dir(repodir, density)
 319             icon_glob = os.path.join(icon_dir, '*.png')
 320             for iconpath in glob.glob(icon_glob):
 321                 resize_icon(iconpath, density)
 322
 323
 324 def scan_apks(apps, apkcache, repodir, knownapks):
 325     """Scan the apks in the given repo directory.
 326
 327     This also extracts the icons.
 328
 329     :param apps: list of all applications, as per metadata.read_metadata
 330     :param apkcache: current apk cache information
 331     :param repodir: repo directory to scan
 332     :param knownapks: known apks info
 333     :returns: (apks, cachechanged) where apks is a list of apk information,
 334               and cachechanged is True if the apkcache got changed.
 335     """
 336
 337     cachechanged = False
 338
 339     icon_dirs = get_icon_dirs(repodir)
 340     for icon_dir in icon_dirs:
 341         if os.path.exists(icon_dir):
 342             if options.clean:
 343                 shutil.rmtree(icon_dir)
 344                 os.makedirs(icon_dir)
 345         else:
 346             os.makedirs(icon_dir)
 347
 348     apks = []
 349     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 350     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 351     vername_pat = re.compile(".*versionName='([^']*)'.*")
 352     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 353     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 354     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 355     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 356     string_pat = re.compile(".*'([^']*)'.*")
 357     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 358
 359         apkfilename = apkfile[len(repodir) + 1:]
 360         if ' ' in apkfilename:
 361             logging.error("No spaces in APK filenames!")
 362             sys.exit(1)
 363
 364         if apkfilename in apkcache:
 365             logging.debug("Reading " + apkfilename + " from cache")
 366             thisinfo = apkcache[apkfilename]
 367
 368         else:
 369
 370             logging.info("Processing " + apkfilename)
 371             thisinfo = {}
 372             thisinfo['apkname'] = apkfilename
 373             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 374             if os.path.exists(os.path.join(repodir, srcfilename)):
 375                 thisinfo['srcname'] = srcfilename
 376             thisinfo['size'] = os.path.getsize(apkfile)
 377             thisinfo['permissions'] = []
 378             thisinfo['features'] = []
 379             thisinfo['icons_src'] = {}
 380             thisinfo['icons'] = {}
 381             p = FDroidPopen([os.path.join(config['sdk_path'], 'build-tools',
 382                                           config['build_tools'], 'aapt'),
 383                              'dump', 'badging', apkfile])
 384             if p.returncode != 0:
 385                 logging.critical("Failed to get apk information")
 386                 sys.exit(1)
 387             for line in p.stdout.splitlines():
 388                 if line.startswith("package:"):
 389                     try:
 390                         thisinfo['id'] = re.match(name_pat, line).group(1)
 391                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 392                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 393                     except Exception, e:
 394                         logging.info("Package matching failed: " + str(e))
 395                         logging.info("Line was: " + line)
 396                         sys.exit(1)
 397                 elif line.startswith("application:"):
 398                     thisinfo['name'] = re.match(label_pat, line).group(1)
 399                     # Keep path to non-dpi icon in case we need it
 400                     match = re.match(icon_pat_nodpi, line)
 401                     if match:
 402                         thisinfo['icons_src']['-1'] = match.group(1)
 403                 elif line.startswith("launchable-activity:"):
 404                     # Only use launchable-activity as fallback to application
 405                     if '-1' not in thisinfo['icons_src']:
 406                         match = re.match(icon_pat_nodpi, line)
 407                         if match:
 408                             thisinfo['icons_src']['-1'] = match.group(1)
 409                 elif line.startswith("application-icon-"):
 410                     match = re.match(icon_pat, line)
 411                     if match:
 412                         density = match.group(1)
 413                         path = match.group(2)
 414                         thisinfo['icons_src'][density] = path
 415                 elif line.startswith("sdkVersion:"):
 416                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 417                 elif line.startswith("maxSdkVersion:"):
 418                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 419                 elif line.startswith("native-code:"):
 420                     thisinfo['nativecode'] = []
 421                     for arch in line[13:].split(' '):
 422                         thisinfo['nativecode'].append(arch[1:-1])
 423                 elif line.startswith("uses-permission:"):
 424                     perm = re.match(string_pat, line).group(1)
 425                     if perm.startswith("android.permission."):
 426                         perm = perm[19:]
 427                     thisinfo['permissions'].append(perm)
 428                 elif line.startswith("uses-feature:"):
 429                     perm = re.match(string_pat, line).group(1)
 430                     #Filter out this, it's only added with the latest SDK tools and
 431                     #causes problems for lots of apps.
 432                     if perm != "android.hardware.screen.portrait" \
 433                             and perm != "android.hardware.screen.landscape":
 434                         if perm.startswith("android.feature."):
 435                             perm = perm[16:]
 436                         thisinfo['features'].append(perm)
 437
 438             if not 'sdkversion' in thisinfo:
 439                 logging.warn("no SDK version information found")
 440                 thisinfo['sdkversion'] = 0
 441
 442             # Check for debuggable apks...
 443             if common.isApkDebuggable(apkfile, config):
 444                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 445
 446             # Calculate the sha256...
 447             sha = hashlib.sha256()
 448             with open(apkfile, 'rb') as f:
 449                 while True:
 450                     t = f.read(1024)
 451                     if len(t) == 0:
 452                         break
 453                     sha.update(t)
 454                 thisinfo['sha256'] = sha.hexdigest()
 455
 456             # Get the signature (or md5 of, to be precise)...
 457             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 458             if not os.path.exists(getsig_dir + "/getsig.class"):
 459                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 460                 sys.exit(1)
 461             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 462                              'getsig', os.path.join(os.getcwd(), apkfile)])
 463             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 464                 logging.critical("Failed to get apk signature")
 465                 sys.exit(1)
 466             thisinfo['sig'] = p.stdout[7:].strip()
 467
 468             apk = zipfile.ZipFile(apkfile, 'r')
 469
 470             iconfilename = "%s.%s.png" % (
 471                 thisinfo['id'],
 472                 thisinfo['versioncode'])
 473
 474             # Extract the icon file...
 475             densities = get_densities()
 476             empty_densities = []
 477             for density in densities:
 478                 if density not in thisinfo['icons_src']:
 479                     empty_densities.append(density)
 480                     continue
 481                 iconsrc = thisinfo['icons_src'][density]
 482                 icon_dir = get_icon_dir(repodir, density)
 483                 icondest = os.path.join(icon_dir, iconfilename)
 484
 485                 try:
 486                     iconfile = open(icondest, 'wb')
 487                     iconfile.write(apk.read(iconsrc))
 488                     iconfile.close()
 489                     thisinfo['icons'][density] = iconfilename
 490
 491                 except:
 492                     logging.warn("Error retrieving icon file")
 493                     del thisinfo['icons'][density]
 494                     del thisinfo['icons_src'][density]
 495                     empty_densities.append(density)
 496
 497             if '-1' in thisinfo['icons_src']:
 498                 iconsrc = thisinfo['icons_src']['-1']
 499                 iconpath = os.path.join(
 500                     get_icon_dir(repodir, None), iconfilename)
 501                 iconfile = open(iconpath, 'wb')
 502                 iconfile.write(apk.read(iconsrc))
 503                 iconfile.close()
 504                 try:
 505                     im = Image.open(iconpath)
 506                     dpi = px_to_dpi(im.size[0])
 507                     for density in densities:
 508                         if density in thisinfo['icons']:
 509                             break
 510                         if density == densities[-1] or dpi >= int(density):
 511                             thisinfo['icons'][density] = iconfilename
 512                             shutil.move(iconpath,
 513                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 514                             empty_densities.remove(density)
 515                             break
 516                 except Exception, e:
 517                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 518
 519             if thisinfo['icons']:
 520                 thisinfo['icon'] = iconfilename
 521
 522             apk.close()
 523
 524             # First try resizing down to not lose quality
 525             last_density = None
 526             for density in densities:
 527                 if density not in empty_densities:
 528                     last_density = density
 529                     continue
 530                 if last_density is None:
 531                     continue
 532                 logging.info("Density %s not available, resizing down from %s"
 533                              % (density, last_density))
 534
 535                 last_iconpath = os.path.join(
 536                     get_icon_dir(repodir, last_density), iconfilename)
 537                 iconpath = os.path.join(
 538                     get_icon_dir(repodir, density), iconfilename)
 539                 try:
 540                     im = Image.open(last_iconpath)
 541                 except:
 542                     logging.warn("Invalid image file at %s" % last_iconpath)
 543                     continue
 544
 545                 size = dpi_to_px(density)
 546
 547                 im.thumbnail((size, size), Image.ANTIALIAS)
 548                 im.save(iconpath, "PNG")
 549                 empty_densities.remove(density)
 550
 551             # Then just copy from the highest resolution available
 552             last_density = None
 553             for density in reversed(densities):
 554                 if density not in empty_densities:
 555                     last_density = density
 556                     continue
 557                 if last_density is None:
 558                     continue
 559                 logging.info("Density %s not available, copying from lower density %s"
 560                              % (density, last_density))
 561
 562                 shutil.copyfile(
 563                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 564                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 565
 566                 empty_densities.remove(density)
 567
 568             for density in densities:
 569                 icon_dir = get_icon_dir(repodir, density)
 570                 icondest = os.path.join(icon_dir, iconfilename)
 571                 resize_icon(icondest, density)
 572
 573             # Copy from icons-mdpi to icons since mdpi is the baseline density
 574             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 575             if os.path.isfile(baseline):
 576                 shutil.copyfile(baseline,
 577                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 578
 579             # Record in known apks, getting the added date at the same time..
 580             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 581             if added:
 582                 thisinfo['added'] = added
 583
 584             apkcache[apkfilename] = thisinfo
 585             cachechanged = True
 586
 587         apks.append(thisinfo)
 588
 589     return apks, cachechanged
 590
 591
 592 repo_pubkey_fingerprint = None
 593
 594
 595 def make_index(apps, apks, repodir, archive, categories):
 596     """Make a repo index.
 597
 598     :param apps: fully populated apps list
 599     :param apks: full populated apks list
 600     :param repodir: the repo directory
 601     :param archive: True if this is the archive repo, False if it's the
 602                     main one.
 603     :param categories: list of categories
 604     """
 605
 606     doc = Document()
 607
 608     def addElement(name, value, doc, parent):
 609         el = doc.createElement(name)
 610         el.appendChild(doc.createTextNode(value))
 611         parent.appendChild(el)
 612
 613     def addElementCDATA(name, value, doc, parent):
 614         el = doc.createElement(name)
 615         el.appendChild(doc.createCDATASection(value))
 616         parent.appendChild(el)
 617
 618     root = doc.createElement("fdroid")
 619     doc.appendChild(root)
 620
 621     repoel = doc.createElement("repo")
 622
 623     if archive:
 624         repoel.setAttribute("name", config['archive_name'])
 625         if config['repo_maxage'] != 0:
 626             repoel.setAttribute("maxage", str(config['repo_maxage']))
 627         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 628         repoel.setAttribute("url", config['archive_url'])
 629         addElement('description', config['archive_description'], doc, repoel)
 630
 631     else:
 632         repoel.setAttribute("name", config['repo_name'])
 633         if config['repo_maxage'] != 0:
 634             repoel.setAttribute("maxage", str(config['repo_maxage']))
 635         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 636         repoel.setAttribute("url", config['repo_url'])
 637         addElement('description', config['repo_description'], doc, repoel)
 638
 639     repoel.setAttribute("version", "12")
 640     repoel.setAttribute("timestamp", str(int(time.time())))
 641
 642     if 'repo_keyalias' in config:
 643
 644         # Generate a certificate fingerprint the same way keytool does it
 645         # (but with slightly different formatting)
 646         def cert_fingerprint(data):
 647             digest = hashlib.sha256(data).digest()
 648             ret = []
 649             ret.append(' '.join("%02X" % ord(b) for b in digest))
 650             return " ".join(ret)
 651
 652         def extract_pubkey():
 653             p = FDroidPopen(['keytool', '-exportcert',
 654                              '-alias', config['repo_keyalias'],
 655                              '-keystore', config['keystore'],
 656                              '-storepass:file', config['keystorepassfile']]
 657                             + config['smartcardoptions'])
 658             if p.returncode != 0:
 659                 msg = "Failed to get repo pubkey!"
 660                 if config['keystore'] == 'NONE':
 661                     msg += ' Is your crypto smartcard plugged in?'
 662                 logging.critical(msg)
 663                 sys.exit(1)
 664             global repo_pubkey_fingerprint
 665             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 666             return "".join("%02x" % ord(b) for b in p.stdout)
 667
 668         repoel.setAttribute("pubkey", extract_pubkey())
 669
 670     root.appendChild(repoel)
 671
 672     for app in apps:
 673
 674         if app['Disabled'] is not None:
 675             continue
 676
 677         # Get a list of the apks for this app...
 678         apklist = []
 679         for apk in apks:
 680             if apk['id'] == app['id']:
 681                 apklist.append(apk)
 682
 683         if len(apklist) == 0:
 684             continue
 685
 686         apel = doc.createElement("application")
 687         apel.setAttribute("id", app['id'])
 688         root.appendChild(apel)
 689
 690         addElement('id', app['id'], doc, apel)
 691         if 'added' in app:
 692             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 693         if 'lastupdated' in app:
 694             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 695         addElement('name', app['Name'], doc, apel)
 696         addElement('summary', app['Summary'], doc, apel)
 697         if app['icon']:
 698             addElement('icon', app['icon'], doc, apel)
 699
 700         def linkres(link):
 701             for app in apps:
 702                 if app['id'] == link:
 703                     return ("fdroid.app:" + link, app['Name'])
 704             raise MetaDataException("Cannot resolve app id " + link)
 705         addElement('desc',
 706                    metadata.description_html(app['Description'], linkres),
 707                    doc, apel)
 708         addElement('license', app['License'], doc, apel)
 709         if 'Categories' in app:
 710             addElement('categories', ','.join(app["Categories"]), doc, apel)
 711             # We put the first (primary) category in LAST, which will have
 712             # the desired effect of making clients that only understand one
 713             # category see that one.
 714             addElement('category', app["Categories"][0], doc, apel)
 715         addElement('web', app['Web Site'], doc, apel)
 716         addElement('source', app['Source Code'], doc, apel)
 717         addElement('tracker', app['Issue Tracker'], doc, apel)
 718         if app['Donate']:
 719             addElement('donate', app['Donate'], doc, apel)
 720         if app['Bitcoin']:
 721             addElement('bitcoin', app['Bitcoin'], doc, apel)
 722         if app['Litecoin']:
 723             addElement('litecoin', app['Litecoin'], doc, apel)
 724         if app['Dogecoin']:
 725             addElement('dogecoin', app['Dogecoin'], doc, apel)
 726         if app['FlattrID']:
 727             addElement('flattr', app['FlattrID'], doc, apel)
 728
 729         # These elements actually refer to the current version (i.e. which
 730         # one is recommended. They are historically mis-named, and need
 731         # changing, but stay like this for now to support existing clients.
 732         addElement('marketversion', app['Current Version'], doc, apel)
 733         addElement('marketvercode', app['Current Version Code'], doc, apel)
 734
 735         if app['AntiFeatures']:
 736             af = app['AntiFeatures'].split(',')
 737             # TODO: Temporarily not including UpstreamNonFree in the index,
 738             # because current F-Droid clients do not understand it, and also
 739             # look ugly when they encounter an unknown antifeature. This
 740             # filtering can be removed in time...
 741             if 'UpstreamNonFree' in af:
 742                 af.remove('UpstreamNonFree')
 743             if af:
 744                 addElement('antifeatures', ','.join(af), doc, apel)
 745         if app['Provides']:
 746             pv = app['Provides'].split(',')
 747             addElement('provides', ','.join(pv), doc, apel)
 748         if app['Requires Root']:
 749             addElement('requirements', 'root', doc, apel)
 750
 751         # Sort the apk list into version order, just so the web site
 752         # doesn't have to do any work by default...
 753         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 754
 755         # Check for duplicates - they will make the client unhappy...
 756         for i in range(len(apklist) - 1):
 757             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 758                 logging.critical("duplicate versions: '%s' - '%s'" % (
 759                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 760                 sys.exit(1)
 761
 762         for apk in apklist:
 763             apkel = doc.createElement("package")
 764             apel.appendChild(apkel)
 765             addElement('version', apk['version'], doc, apkel)
 766             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 767             addElement('apkname', apk['apkname'], doc, apkel)
 768             if 'srcname' in apk:
 769                 addElement('srcname', apk['srcname'], doc, apkel)
 770             for hash_type in ['sha256']:
 771                 if not hash_type in apk:
 772                     continue
 773                 hashel = doc.createElement("hash")
 774                 hashel.setAttribute("type", hash_type)
 775                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 776                 apkel.appendChild(hashel)
 777             addElement('sig', apk['sig'], doc, apkel)
 778             addElement('size', str(apk['size']), doc, apkel)
 779             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 780             if 'maxsdkversion' in apk:
 781                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 782             if 'added' in apk:
 783                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 784             if app['Requires Root']:
 785                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 786                     apk['permissions'].append('ACCESS_SUPERUSER')
 787
 788             if len(apk['permissions']) > 0:
 789                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 790             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 791                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 792             if len(apk['features']) > 0:
 793                 addElement('features', ','.join(apk['features']), doc, apkel)
 794
 795     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 796     if options.pretty:
 797         output = doc.toprettyxml()
 798     else:
 799         output = doc.toxml()
 800     of.write(output)
 801     of.close()
 802
 803     if 'repo_keyalias' in config:
 804
 805         logging.info("Creating signed index with this key:")
 806         logging.info("SHA256: %s" % repo_pubkey_fingerprint)
 807
 808         #Create a jar of the index...
 809         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 810         if p.returncode != 0:
 811             logging.critical("Failed to create jar file")
 812             sys.exit(1)
 813
 814         # Sign the index...
 815         args = ['jarsigner', '-keystore', config['keystore'],
 816                 '-storepass:file', config['keystorepassfile'],
 817                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 818                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 819         if config['keystore'] == 'NONE':
 820             args += config['smartcardoptions']
 821         else:  # smardcards never use -keypass
 822             args += ['-keypass:file', config['keypassfile']]
 823         p = FDroidPopen(args)
 824         # TODO keypass should be sent via stdin
 825         if p.returncode != 0:
 826             logging.info("Failed to sign index")
 827             sys.exit(1)
 828
 829     # Copy the repo icon into the repo directory...
 830     icon_dir = os.path.join(repodir, 'icons')
 831     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 832     shutil.copyfile(config['repo_icon'], iconfilename)
 833
 834     # Write a category list in the repo to allow quick access...
 835     catdata = ''
 836     for cat in categories:
 837         catdata += cat + '\n'
 838     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 839     f.write(catdata)
 840     f.close()
 841
 842
 843 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 844
 845     for app in apps:
 846
 847         # Get a list of the apks for this app...
 848         apklist = []
 849         for apk in apks:
 850             if apk['id'] == app['id']:
 851                 apklist.append(apk)
 852
 853         # Sort the apk list into version order...
 854         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 855
 856         if app['Archive Policy']:
 857             keepversions = int(app['Archive Policy'][:-9])
 858         else:
 859             keepversions = defaultkeepversions
 860
 861         if len(apklist) > keepversions:
 862             for apk in apklist[keepversions:]:
 863                 logging.info("Moving " + apk['apkname'] + " to archive")
 864                 shutil.move(os.path.join(repodir, apk['apkname']),
 865                             os.path.join(archivedir, apk['apkname']))
 866                 if 'srcname' in apk:
 867                     shutil.move(os.path.join(repodir, apk['srcname']),
 868                                 os.path.join(archivedir, apk['srcname']))
 869                 archapks.append(apk)
 870                 apks.remove(apk)
 871
 872
 873 config = None
 874 options = None
 875
 876
 877 def main():
 878
 879     global config, options
 880
 881     # Parse command line...
 882     parser = OptionParser()
 883     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 884                       help="Create skeleton metadata files that are missing")
 885     parser.add_option("-v", "--verbose", action="store_true", default=False,
 886                       help="Spew out even more information than normal")
 887     parser.add_option("-q", "--quiet", action="store_true", default=False,
 888                       help="Restrict output to warnings and errors")
 889     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 890                       help="Report on build data status")
 891     parser.add_option("-i", "--interactive", default=False, action="store_true",
 892                       help="Interactively ask about things that need updating.")
 893     parser.add_option("-I", "--icons", action="store_true", default=False,
 894                       help="Resize all the icons exceeding the max pixel size and exit")
 895     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 896                       help="Specify editor to use in interactive mode. Default " +
 897                       "is /etc/alternatives/editor")
 898     parser.add_option("-w", "--wiki", default=False, action="store_true",
 899                       help="Update the wiki")
 900     parser.add_option("", "--pretty", action="store_true", default=False,
 901                       help="Produce human-readable index.xml")
 902     parser.add_option("--clean", action="store_true", default=False,
 903                       help="Clean update - don't uses caches, reprocess all apks")
 904     (options, args) = parser.parse_args()
 905
 906     config = common.read_config(options)
 907
 908     repodirs = ['repo']
 909     if config['archive_older'] != 0:
 910         repodirs.append('archive')
 911         if not os.path.exists('archive'):
 912             os.mkdir('archive')
 913
 914     if options.icons:
 915         resize_all_icons(repodirs)
 916         sys.exit(0)
 917
 918     # Get all apps...
 919     apps = metadata.read_metadata()
 920
 921     # Generate a list of categories...
 922     categories = set()
 923     for app in apps:
 924         categories.update(app['Categories'])
 925
 926     # Read known apks data (will be updated and written back when we've finished)
 927     knownapks = common.KnownApks()
 928
 929     # Gather information about all the apk files in the repo directory, using
 930     # cached data if possible.
 931     apkcachefile = os.path.join('tmp', 'apkcache')
 932     if not options.clean and os.path.exists(apkcachefile):
 933         with open(apkcachefile, 'rb') as cf:
 934             apkcache = pickle.load(cf)
 935     else:
 936         apkcache = {}
 937     cachechanged = False
 938
 939     delete_disabled_builds(apps, apkcache, repodirs)
 940
 941     # Scan all apks in the main repo
 942     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 943     if cc:
 944         cachechanged = True
 945
 946     # Scan the archive repo for apks as well
 947     if len(repodirs) > 1:
 948         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 949         if cc:
 950             cachechanged = True
 951     else:
 952         archapks = []
 953
 954     # Some information from the apks needs to be applied up to the application
 955     # level. When doing this, we use the info from the most recent version's apk.
 956     # We deal with figuring out when the app was added and last updated at the
 957     # same time.
 958     for app in apps:
 959         bestver = 0
 960         added = None
 961         lastupdated = None
 962         for apk in apks + archapks:
 963             if apk['id'] == app['id']:
 964                 if apk['versioncode'] > bestver:
 965                     bestver = apk['versioncode']
 966                     bestapk = apk
 967
 968                 if 'added' in apk:
 969                     if not added or apk['added'] < added:
 970                         added = apk['added']
 971                     if not lastupdated or apk['added'] > lastupdated:
 972                         lastupdated = apk['added']
 973
 974         if added:
 975             app['added'] = added
 976         else:
 977             logging.warn("Don't know when " + app['id'] + " was added")
 978         if lastupdated:
 979             app['lastupdated'] = lastupdated
 980         else:
 981             logging.warn("Don't know when " + app['id'] + " was last updated")
 982
 983         if bestver == 0:
 984             if app['Name'] is None:
 985                 app['Name'] = app['id']
 986             app['icon'] = None
 987             logging.warn("Application " + app['id'] + " has no packages")
 988         else:
 989             if app['Name'] is None:
 990                 app['Name'] = bestapk['name']
 991             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 992
 993     # Sort the app list by name, then the web site doesn't have to by default.
 994     # (we had to wait until we'd scanned the apks to do this, because mostly the
 995     # name comes from there!)
 996     apps = sorted(apps, key=lambda app: app['Name'].upper())
 997
 998     # Generate warnings for apk's with no metadata (or create skeleton
 999     # metadata files, if requested on the command line)
1000     for apk in apks:
1001         found = False
1002         for app in apps:
1003             if app['id'] == apk['id']:
1004                 found = True
1005                 break
1006         if not found:
1007             if options.createmeta:
1008                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1009                 f.write("License:Unknown\n")
1010                 f.write("Web Site:\n")
1011                 f.write("Source Code:\n")
1012                 f.write("Issue Tracker:\n")
1013                 f.write("Summary:" + apk['name'] + "\n")
1014                 f.write("Description:\n")
1015                 f.write(apk['name'] + "\n")
1016                 f.write(".\n")
1017                 f.close()
1018                 logging.info("Generated skeleton metadata for " + apk['id'])
1019             else:
1020                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata - removing")
1021                 rmf = os.path.join(repodirs[0], apk['apkname'])
1022                 if not os.path.exists(rmf):
1023                     logging.error("Could not find {0} to remove it".format(rmf))
1024                 else:
1025                     os.remove(rmf)
1026
1027     if len(repodirs) > 1:
1028         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1029
1030     # Make the index for the main repo...
1031     make_index(apps, apks, repodirs[0], False, categories)
1032
1033     # If there's an archive repo,  make the index for it. We already scanned it
1034     # earlier on.
1035     if len(repodirs) > 1:
1036         make_index(apps, archapks, repodirs[1], True, categories)
1037
1038     if config['update_stats']:
1039
1040         # Update known apks info...
1041         knownapks.writeifchanged()
1042
1043         # Generate latest apps data for widget
1044         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1045             data = ''
1046             for line in file(os.path.join('stats', 'latestapps.txt')):
1047                 appid = line.rstrip()
1048                 data += appid + "\t"
1049                 for app in apps:
1050                     if app['id'] == appid:
1051                         data += app['Name'] + "\t"
1052                         if app['icon'] is not None:
1053                             data += app['icon'] + "\t"
1054                         data += app['License'] + "\n"
1055                         break
1056             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1057             f.write(data)
1058             f.close()
1059
1060     if cachechanged:
1061         with open(apkcachefile, 'wb') as cf:
1062             pickle.dump(apkcache, cf)
1063
1064     # Update the wiki...
1065     if options.wiki:
1066         update_wiki(apps, apks + archapks)
1067
1068     logging.info("Finished.")
1069
1070 if __name__ == "__main__":
1071     main()