fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76             path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88                 app['id'],
  89                 app['Name'],
  90                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92                 app['Source Code'],
  93                 app['Issue Tracker'],
  94                 app['Web Site'],
  95                 app['Donate'],
  96                 app['FlattrID'],
  97                 app['Bitcoin'],
  98                 app['Litecoin'],
  99                 app['Dogecoin'],
 100                 app['License'],
 101                 app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if 'disable' in thisbuild:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 apklist.append({
 133                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 134                         'versioncode': int(thisbuild['vercode']),
 135                         'version': thisbuild['version'],
 136                         'buildproblem': thisbuild['disable']
 137                     })
 138             else:
 139                 builtit = False
 140                 for apk in apklist:
 141                     if apk['versioncode'] == int(thisbuild['vercode']):
 142                         builtit = True
 143                         break
 144                 if not builtit:
 145                     buildfails = True
 146                     apklist.append({
 147                             'versioncode': int(thisbuild['vercode']),
 148                             'version': thisbuild['version'],
 149                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 150                         })
 151         if app['Current Version Code'] == '0':
 152             cantupdate = True
 153         # Sort with most recent first...
 154         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 155
 156         wikidata += "=Versions=\n"
 157         if len(apklist) == 0:
 158             wikidata += "We currently have no versions of this app available."
 159         elif not gotcurrentver:
 160             wikidata += "We don't have the current version of this app."
 161         else:
 162             wikidata += "We have the current version of this app."
 163         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 164         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 165         if len(app['No Source Since']) > 0:
 166             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 167         if len(app['Current Version']) > 0:
 168             wikidata += "The current (recommended) version is " + app['Current Version']
 169             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 170         validapks = 0
 171         for apk in apklist:
 172             wikidata += "==" + apk['version'] + "==\n"
 173
 174             if 'buildproblem' in apk:
 175                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 176             else:
 177                 validapks += 1
 178                 wikidata += "This version is built and signed by "
 179                 if 'srcname' in apk:
 180                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 181                 else:
 182                     wikidata += "the original developer.\n\n"
 183             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 184
 185         wikidata += '\n[[Category:' + wikicat + ']]\n'
 186         if len(app['No Source Since']) > 0:
 187             wikidata += '\n[[Category:Apps missing source code]]\n'
 188         if validapks == 0 and not app['Disabled']:
 189             wikidata += '\n[[Category:Apps with no packages]]\n'
 190         if cantupdate and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps we can't update]]\n"
 192         if buildfails and not app['Disabled']:
 193             wikidata += "\n[[Category:Apps with failing builds]]\n"
 194         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 195             wikidata += '\n[[Category:Apps to Update]]\n'
 196         if app['Disabled']:
 197             wikidata += '\n[[Category:Apps that are disabled]]\n'
 198         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 199             wikidata += '\n[[Category:Apps with no update check]]\n'
 200         for appcat in app['Categories']:
 201             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 202
 203         # We can't have underscores in the page name, even if they're in
 204         # the package ID, because MediaWiki messes with them...
 205         pagename = app['id'].replace('_', ' ')
 206
 207         # Drop a trailing newline, because mediawiki is going to drop it anyway
 208         # and it we don't we'll think the page has changed when it hasn't...
 209         if wikidata.endswith('\n'):
 210             wikidata = wikidata[:-1]
 211
 212         generated_pages[pagename] = wikidata
 213
 214         # Make a redirect from the name to the ID too, unless there's
 215         # already an existing page with the name and it isn't a redirect.
 216         noclobber = False
 217         apppagename = app['Name'].replace('_', ' ')
 218         apppagename = apppagename.replace('{', '')
 219         apppagename = apppagename.replace('}', ' ')
 220         apppagename = apppagename.replace(':', ' ')
 221         # Drop double spaces caused mostly by replacing ':' above
 222         apppagename = apppagename.replace('  ', ' ')
 223         for expagename in site.allpages(prefix=apppagename,
 224                 filterredir='nonredirects', generator=False):
 225             if expagename == apppagename:
 226                 noclobber = True
 227         # Another reason not to make the redirect page is if the app name
 228         # is the same as it's ID, because that will overwrite the real page
 229         # with an redirect to itself! (Although it seems like an odd
 230         # scenario this happens a lot, e.g. where there is metadata but no
 231         # builds or binaries to extract a name from.
 232         if apppagename == pagename:
 233             noclobber = True
 234         if not noclobber:
 235             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 236
 237     for tcat, genp in [(wikicat, generated_pages),
 238             (wikiredircat, generated_redirects)]:
 239         catpages = site.Pages['Category:' + tcat]
 240         existingpages = []
 241         for page in catpages:
 242             existingpages.append(page.name)
 243             if page.name in genp:
 244                 pagetxt = page.edit()
 245                 if pagetxt != genp[page.name]:
 246                     logging.debug("Updating modified page " + page.name)
 247                     page.save(genp[page.name], summary='Auto-updated')
 248                 else:
 249                     logging.debug("Page " + page.name + " is unchanged")
 250             else:
 251                 logging.warn("Deleting page " + page.name)
 252                 page.delete('No longer published')
 253         for pagename, text in genp.items():
 254             logging.debug("Checking " + pagename)
 255             if not pagename in existingpages:
 256                 logging.debug("Creating page " + pagename)
 257                 try:
 258                     newpage = site.Pages[pagename]
 259                     newpage.save(text, summary='Auto-created')
 260                 except:
 261                     logging.error("...FAILED to create page")
 262
 263     # Purge server cache to ensure counts are up to date
 264     site.pages['Repository Maintenance'].purge()
 265
 266
 267 def delete_disabled_builds(apps, apkcache, repodirs):
 268     """Delete disabled build outputs.
 269
 270     :param apps: list of all applications, as per metadata.read_metadata
 271     :param apkcache: current apk cache information
 272     :param repodirs: the repo directories to process
 273     """
 274     for app in apps:
 275         for build in app['builds']:
 276             if 'disable' in build:
 277                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 278                 for repodir in repodirs:
 279                     apkpath = os.path.join(repodir, apkfilename)
 280                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 281                     for name in [apkpath, srcpath]:
 282                         if os.path.exists(name):
 283                             logging.warn("Deleting disabled build output " + apkfilename)
 284                             os.remove(name)
 285                 if apkfilename in apkcache:
 286                     del apkcache[apkfilename]
 287
 288
 289 def resize_icon(iconpath, density):
 290
 291     if not os.path.isfile(iconpath):
 292         return
 293
 294     try:
 295         im = Image.open(iconpath)
 296         size = dpi_to_px(density)
 297
 298         if any(length > size for length in im.size):
 299             oldsize = im.size
 300             im.thumbnail((size, size), Image.ANTIALIAS)
 301             logging.debug("%s was too large at %s - new size is %s" % (
 302                 iconpath, oldsize, im.size))
 303             im.save(iconpath, "PNG")
 304
 305         else:
 306             logging.info("%s is small enough: %s" % im.size)
 307
 308     except Exception, e:
 309         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 310
 311
 312 def resize_all_icons(repodirs):
 313     """Resize all icons that exceed the max size
 314
 315     :param repodirs: the repo directories to process
 316     """
 317     for repodir in repodirs:
 318         for density in get_densities():
 319             icon_dir = get_icon_dir(repodir, density)
 320             icon_glob = os.path.join(icon_dir, '*.png')
 321             for iconpath in glob.glob(icon_glob):
 322                 resize_icon(iconpath, density)
 323
 324
 325 def scan_apks(apps, apkcache, repodir, knownapks):
 326     """Scan the apks in the given repo directory.
 327
 328     This also extracts the icons.
 329
 330     :param apps: list of all applications, as per metadata.read_metadata
 331     :param apkcache: current apk cache information
 332     :param repodir: repo directory to scan
 333     :param knownapks: known apks info
 334     :returns: (apks, cachechanged) where apks is a list of apk information,
 335               and cachechanged is True if the apkcache got changed.
 336     """
 337
 338     cachechanged = False
 339
 340     icon_dirs = get_icon_dirs(repodir)
 341     for icon_dir in icon_dirs:
 342         if os.path.exists(icon_dir):
 343             if options.clean:
 344                 shutil.rmtree(icon_dir)
 345                 os.makedirs(icon_dir)
 346         else:
 347             os.makedirs(icon_dir)
 348
 349     apks = []
 350     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 351     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 352     vername_pat = re.compile(".*versionName='([^']*)'.*")
 353     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 354     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 355     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 356     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 357     string_pat = re.compile(".*'([^']*)'.*")
 358     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 359
 360         apkfilename = apkfile[len(repodir) + 1:]
 361         if ' ' in apkfilename:
 362             logging.error("No spaces in APK filenames!")
 363             sys.exit(1)
 364
 365         if apkfilename in apkcache:
 366             logging.debug("Reading " + apkfilename + " from cache")
 367             thisinfo = apkcache[apkfilename]
 368
 369         else:
 370
 371             logging.info("Processing " + apkfilename)
 372             thisinfo = {}
 373             thisinfo['apkname'] = apkfilename
 374             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 375             if os.path.exists(os.path.join(repodir, srcfilename)):
 376                 thisinfo['srcname'] = srcfilename
 377             thisinfo['size'] = os.path.getsize(apkfile)
 378             thisinfo['permissions'] = []
 379             thisinfo['features'] = []
 380             thisinfo['icons_src'] = {}
 381             thisinfo['icons'] = {}
 382             p = FDroidPopen([os.path.join(config['sdk_path'],
 383                         'build-tools', config['build_tools'], 'aapt'),
 384                     'dump', 'badging', apkfile])
 385             if p.returncode != 0:
 386                 logging.critical("Failed to get apk information")
 387                 sys.exit(1)
 388             for line in p.stdout.splitlines():
 389                 if line.startswith("package:"):
 390                     try:
 391                         thisinfo['id'] = re.match(name_pat, line).group(1)
 392                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 393                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 394                     except Exception, e:
 395                         logging.info("Package matching failed: " + str(e))
 396                         logging.info("Line was: " + line)
 397                         sys.exit(1)
 398                 elif line.startswith("application:"):
 399                     thisinfo['name'] = re.match(label_pat, line).group(1)
 400                     # Keep path to non-dpi icon in case we need it
 401                     match = re.match(icon_pat_nodpi, line)
 402                     if match:
 403                         thisinfo['icons_src']['-1'] = match.group(1)
 404                 elif line.startswith("launchable-activity:"):
 405                     # Only use launchable-activity as fallback to application
 406                     if '-1' not in thisinfo['icons_src']:
 407                         match = re.match(icon_pat_nodpi, line)
 408                         if match:
 409                             thisinfo['icons_src']['-1'] = match.group(1)
 410                 elif line.startswith("application-icon-"):
 411                     match = re.match(icon_pat, line)
 412                     if match:
 413                         density = match.group(1)
 414                         path = match.group(2)
 415                         thisinfo['icons_src'][density] = path
 416                 elif line.startswith("sdkVersion:"):
 417                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 418                 elif line.startswith("maxSdkVersion:"):
 419                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 420                 elif line.startswith("native-code:"):
 421                     thisinfo['nativecode'] = []
 422                     for arch in line[13:].split(' '):
 423                         thisinfo['nativecode'].append(arch[1:-1])
 424                 elif line.startswith("uses-permission:"):
 425                     perm = re.match(string_pat, line).group(1)
 426                     if perm.startswith("android.permission."):
 427                         perm = perm[19:]
 428                     thisinfo['permissions'].append(perm)
 429                 elif line.startswith("uses-feature:"):
 430                     perm = re.match(string_pat, line).group(1)
 431                     #Filter out this, it's only added with the latest SDK tools and
 432                     #causes problems for lots of apps.
 433                     if (perm != "android.hardware.screen.portrait" and
 434                         perm != "android.hardware.screen.landscape"):
 435                         if perm.startswith("android.feature."):
 436                             perm = perm[16:]
 437                         thisinfo['features'].append(perm)
 438
 439             if not 'sdkversion' in thisinfo:
 440                 logging.warn("no SDK version information found")
 441                 thisinfo['sdkversion'] = 0
 442
 443             # Check for debuggable apks...
 444             if common.isApkDebuggable(apkfile, config):
 445                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 446
 447             # Calculate the sha256...
 448             sha = hashlib.sha256()
 449             with open(apkfile, 'rb') as f:
 450                 while True:
 451                     t = f.read(1024)
 452                     if len(t) == 0:
 453                         break
 454                     sha.update(t)
 455                 thisinfo['sha256'] = sha.hexdigest()
 456
 457             # Get the signature (or md5 of, to be precise)...
 458             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 459             if not os.path.exists(getsig_dir + "/getsig.class"):
 460                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 461                 sys.exit(1)
 462             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 463                         'getsig', os.path.join(os.getcwd(), apkfile)])
 464             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 465                 logging.critical("Failed to get apk signature")
 466                 sys.exit(1)
 467             thisinfo['sig'] = p.stdout[7:].strip()
 468
 469             apk = zipfile.ZipFile(apkfile, 'r')
 470
 471             iconfilename = "%s.%s.png" % (
 472                     thisinfo['id'],
 473                     thisinfo['versioncode'])
 474
 475             # Extract the icon file...
 476             densities = get_densities()
 477             empty_densities = []
 478             for density in densities:
 479                 if density not in thisinfo['icons_src']:
 480                     empty_densities.append(density)
 481                     continue
 482                 iconsrc = thisinfo['icons_src'][density]
 483                 icon_dir = get_icon_dir(repodir, density)
 484                 icondest = os.path.join(icon_dir, iconfilename)
 485
 486                 try:
 487                     iconfile = open(icondest, 'wb')
 488                     iconfile.write(apk.read(iconsrc))
 489                     iconfile.close()
 490                     thisinfo['icons'][density] = iconfilename
 491
 492                 except:
 493                     logging.warn("Error retrieving icon file")
 494                     del thisinfo['icons'][density]
 495                     del thisinfo['icons_src'][density]
 496                     empty_densities.append(density)
 497
 498             if '-1' in thisinfo['icons_src']:
 499                 iconsrc = thisinfo['icons_src']['-1']
 500                 iconpath = os.path.join(
 501                         get_icon_dir(repodir, None), iconfilename)
 502                 iconfile = open(iconpath, 'wb')
 503                 iconfile.write(apk.read(iconsrc))
 504                 iconfile.close()
 505                 try:
 506                     im = Image.open(iconpath)
 507                     dpi = px_to_dpi(im.size[0])
 508                     for density in densities:
 509                         if density in thisinfo['icons']:
 510                             break
 511                         if density == densities[-1] or dpi >= int(density):
 512                             thisinfo['icons'][density] = iconfilename
 513                             shutil.move(iconpath,
 514                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 515                             empty_densities.remove(density)
 516                             break
 517                 except Exception, e:
 518                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 519
 520             if thisinfo['icons']:
 521                 thisinfo['icon'] = iconfilename
 522
 523             apk.close()
 524
 525             # First try resizing down to not lose quality
 526             last_density = None
 527             for density in densities:
 528                 if density not in empty_densities:
 529                     last_density = density
 530                     continue
 531                 if last_density is None:
 532                     continue
 533                 logging.info("Density %s not available, resizing down from %s" % (
 534                         density, last_density))
 535
 536                 last_iconpath = os.path.join(
 537                         get_icon_dir(repodir, last_density), iconfilename)
 538                 iconpath = os.path.join(
 539                         get_icon_dir(repodir, density), iconfilename)
 540                 try:
 541                     im = Image.open(last_iconpath)
 542                 except:
 543                     logging.warn("Invalid image file at %s" % last_iconpath)
 544                     continue
 545
 546                 size = dpi_to_px(density)
 547
 548                 im.thumbnail((size, size), Image.ANTIALIAS)
 549                 im.save(iconpath, "PNG")
 550                 empty_densities.remove(density)
 551
 552             # Then just copy from the highest resolution available
 553             last_density = None
 554             for density in reversed(densities):
 555                 if density not in empty_densities:
 556                     last_density = density
 557                     continue
 558                 if last_density is None:
 559                     continue
 560                 logging.info("Density %s not available, copying from lower density %s" % (
 561                         density, last_density))
 562
 563                 shutil.copyfile(
 564                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 565                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 566
 567                 empty_densities.remove(density)
 568
 569             for density in densities:
 570                 icon_dir = get_icon_dir(repodir, density)
 571                 icondest = os.path.join(icon_dir, iconfilename)
 572                 resize_icon(icondest, density)
 573
 574             # Copy from icons-mdpi to icons since mdpi is the baseline density
 575             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 576             if os.path.isfile(baseline):
 577                 shutil.copyfile(baseline,
 578                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 579
 580             # Record in known apks, getting the added date at the same time..
 581             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 582             if added:
 583                 thisinfo['added'] = added
 584
 585             apkcache[apkfilename] = thisinfo
 586             cachechanged = True
 587
 588         apks.append(thisinfo)
 589
 590     return apks, cachechanged
 591
 592
 593 repo_pubkey_fingerprint = None
 594
 595
 596 def make_index(apps, apks, repodir, archive, categories):
 597     """Make a repo index.
 598
 599     :param apps: fully populated apps list
 600     :param apks: full populated apks list
 601     :param repodir: the repo directory
 602     :param archive: True if this is the archive repo, False if it's the
 603                     main one.
 604     :param categories: list of categories
 605     """
 606
 607     doc = Document()
 608
 609     def addElement(name, value, doc, parent):
 610         el = doc.createElement(name)
 611         el.appendChild(doc.createTextNode(value))
 612         parent.appendChild(el)
 613
 614     def addElementCDATA(name, value, doc, parent):
 615         el = doc.createElement(name)
 616         el.appendChild(doc.createCDATASection(value))
 617         parent.appendChild(el)
 618
 619     root = doc.createElement("fdroid")
 620     doc.appendChild(root)
 621
 622     repoel = doc.createElement("repo")
 623
 624     if archive:
 625         repoel.setAttribute("name", config['archive_name'])
 626         if config['repo_maxage'] != 0:
 627             repoel.setAttribute("maxage", str(config['repo_maxage']))
 628         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 629         repoel.setAttribute("url", config['archive_url'])
 630         addElement('description', config['archive_description'], doc, repoel)
 631
 632     else:
 633         repoel.setAttribute("name", config['repo_name'])
 634         if config['repo_maxage'] != 0:
 635             repoel.setAttribute("maxage", str(config['repo_maxage']))
 636         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 637         repoel.setAttribute("url", config['repo_url'])
 638         addElement('description', config['repo_description'], doc, repoel)
 639
 640     repoel.setAttribute("version", "12")
 641     repoel.setAttribute("timestamp", str(int(time.time())))
 642
 643     if 'repo_keyalias' in config:
 644
 645         # Generate a certificate fingerprint the same way keytool does it
 646         # (but with slightly different formatting)
 647         def cert_fingerprint(data):
 648             digest = hashlib.sha256(data).digest()
 649             ret = []
 650             ret.append(' '.join("%02X" % ord(b) for b in digest))
 651             return " ".join(ret)
 652
 653         def extract_pubkey():
 654             p = FDroidPopen(['keytool', '-exportcert',
 655                                   '-alias', config['repo_keyalias'],
 656                                   '-keystore', config['keystore'],
 657                                   '-storepass:file', config['keystorepassfile']]
 658                             + config['smartcardoptions'])
 659             if p.returncode != 0:
 660                 msg = "Failed to get repo pubkey!"
 661                 if config['keystore'] == 'NONE':
 662                     msg += ' Is your crypto smartcard plugged in?'
 663                 logging.critical(msg)
 664                 sys.exit(1)
 665             global repo_pubkey_fingerprint
 666             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 667             return "".join("%02x" % ord(b) for b in p.stdout)
 668
 669         repoel.setAttribute("pubkey", extract_pubkey())
 670
 671     root.appendChild(repoel)
 672
 673     for app in apps:
 674
 675         if app['Disabled'] is not None:
 676             continue
 677
 678         # Get a list of the apks for this app...
 679         apklist = []
 680         for apk in apks:
 681             if apk['id'] == app['id']:
 682                 apklist.append(apk)
 683
 684         if len(apklist) == 0:
 685             continue
 686
 687         apel = doc.createElement("application")
 688         apel.setAttribute("id", app['id'])
 689         root.appendChild(apel)
 690
 691         addElement('id', app['id'], doc, apel)
 692         if 'added' in app:
 693             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 694         if 'lastupdated' in app:
 695             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 696         addElement('name', app['Name'], doc, apel)
 697         addElement('summary', app['Summary'], doc, apel)
 698         if app['icon']:
 699             addElement('icon', app['icon'], doc, apel)
 700
 701         def linkres(link):
 702             for app in apps:
 703                 if app['id'] == link:
 704                     return ("fdroid.app:" + link, app['Name'])
 705             raise MetaDataException("Cannot resolve app id " + link)
 706         addElement('desc',
 707                 metadata.description_html(app['Description'], linkres), doc, apel)
 708         addElement('license', app['License'], doc, apel)
 709         if 'Categories' in app:
 710             addElement('categories', ','.join(app["Categories"]), doc, apel)
 711             # We put the first (primary) category in LAST, which will have
 712             # the desired effect of making clients that only understand one
 713             # category see that one.
 714             addElement('category', app["Categories"][0], doc, apel)
 715         addElement('web', app['Web Site'], doc, apel)
 716         addElement('source', app['Source Code'], doc, apel)
 717         addElement('tracker', app['Issue Tracker'], doc, apel)
 718         if app['Donate']:
 719             addElement('donate', app['Donate'], doc, apel)
 720         if app['Bitcoin']:
 721             addElement('bitcoin', app['Bitcoin'], doc, apel)
 722         if app['Litecoin']:
 723             addElement('litecoin', app['Litecoin'], doc, apel)
 724         if app['Dogecoin']:
 725             addElement('dogecoin', app['Dogecoin'], doc, apel)
 726         if app['FlattrID']:
 727             addElement('flattr', app['FlattrID'], doc, apel)
 728
 729         # These elements actually refer to the current version (i.e. which
 730         # one is recommended. They are historically mis-named, and need
 731         # changing, but stay like this for now to support existing clients.
 732         addElement('marketversion', app['Current Version'], doc, apel)
 733         addElement('marketvercode', app['Current Version Code'], doc, apel)
 734
 735         if app['AntiFeatures']:
 736             af = app['AntiFeatures'].split(',')
 737             # TODO: Temporarily not including UpstreamNonFree in the index,
 738             # because current F-Droid clients do not understand it, and also
 739             # look ugly when they encounter an unknown antifeature. This
 740             # filtering can be removed in time...
 741             if 'UpstreamNonFree' in af:
 742                 af.remove('UpstreamNonFree')
 743             if af:
 744                 addElement('antifeatures', ','.join(af), doc, apel)
 745         if app['Provides']:
 746             pv = app['Provides'].split(',')
 747             addElement('provides', ','.join(pv), doc, apel)
 748         if app['Requires Root']:
 749             addElement('requirements', 'root', doc, apel)
 750
 751         # Sort the apk list into version order, just so the web site
 752         # doesn't have to do any work by default...
 753         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 754
 755         # Check for duplicates - they will make the client unhappy...
 756         for i in range(len(apklist) - 1):
 757             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 758                 logging.critical("duplicate versions: '%s' - '%s'" % (
 759                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 760                 sys.exit(1)
 761
 762         for apk in apklist:
 763             apkel = doc.createElement("package")
 764             apel.appendChild(apkel)
 765             addElement('version', apk['version'], doc, apkel)
 766             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 767             addElement('apkname', apk['apkname'], doc, apkel)
 768             if 'srcname' in apk:
 769                 addElement('srcname', apk['srcname'], doc, apkel)
 770             for hash_type in ['sha256']:
 771                 if not hash_type in apk:
 772                     continue
 773                 hashel = doc.createElement("hash")
 774                 hashel.setAttribute("type", hash_type)
 775                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 776                 apkel.appendChild(hashel)
 777             addElement('sig', apk['sig'], doc, apkel)
 778             addElement('size', str(apk['size']), doc, apkel)
 779             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 780             if 'maxsdkversion' in apk:
 781                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 782             if 'added' in apk:
 783                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 784             if app['Requires Root']:
 785                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 786                     apk['permissions'].append('ACCESS_SUPERUSER')
 787
 788             if len(apk['permissions']) > 0:
 789                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 790             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 791                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 792             if len(apk['features']) > 0:
 793                 addElement('features', ','.join(apk['features']), doc, apkel)
 794
 795     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 796     if options.pretty:
 797         output = doc.toprettyxml()
 798     else:
 799         output = doc.toxml()
 800     of.write(output)
 801     of.close()
 802
 803     if 'repo_keyalias' in config:
 804
 805         logging.info("Creating signed index with this key:")
 806         logging.info("SHA256: %s" % repo_pubkey_fingerprint)
 807
 808         #Create a jar of the index...
 809         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 810         if p.returncode != 0:
 811             logging.critical("Failed to create jar file")
 812             sys.exit(1)
 813
 814         # Sign the index...
 815         args = ['jarsigner', '-keystore', config['keystore'],
 816                 '-storepass:file', config['keystorepassfile'],
 817                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 818                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 819         if config['keystore'] == 'NONE':
 820             args += config['smartcardoptions']
 821         else:  # smardcards never use -keypass
 822             args += ['-keypass:file', config['keypassfile']]
 823         p = FDroidPopen(args)
 824         # TODO keypass should be sent via stdin
 825         if p.returncode != 0:
 826             logging.info("Failed to sign index")
 827             sys.exit(1)
 828
 829     # Copy the repo icon into the repo directory...
 830     icon_dir = os.path.join(repodir, 'icons')
 831     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 832     shutil.copyfile(config['repo_icon'], iconfilename)
 833
 834     # Write a category list in the repo to allow quick access...
 835     catdata = ''
 836     for cat in categories:
 837         catdata += cat + '\n'
 838     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 839     f.write(catdata)
 840     f.close()
 841
 842
 843 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 844
 845     for app in apps:
 846
 847         # Get a list of the apks for this app...
 848         apklist = []
 849         for apk in apks:
 850             if apk['id'] == app['id']:
 851                 apklist.append(apk)
 852
 853         # Sort the apk list into version order...
 854         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 855
 856         if app['Archive Policy']:
 857             keepversions = int(app['Archive Policy'][:-9])
 858         else:
 859             keepversions = defaultkeepversions
 860
 861         if len(apklist) > keepversions:
 862             for apk in apklist[keepversions:]:
 863                 logging.info("Moving " + apk['apkname'] + " to archive")
 864                 shutil.move(os.path.join(repodir, apk['apkname']),
 865                     os.path.join(archivedir, apk['apkname']))
 866                 if 'srcname' in apk:
 867                     shutil.move(os.path.join(repodir, apk['srcname']),
 868                         os.path.join(archivedir, apk['srcname']))
 869                 archapks.append(apk)
 870                 apks.remove(apk)
 871
 872
 873 config = None
 874 options = None
 875
 876
 877 def main():
 878
 879     global config, options
 880
 881     # Parse command line...
 882     parser = OptionParser()
 883     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 884                       help="Create skeleton metadata files that are missing")
 885     parser.add_option("-v", "--verbose", action="store_true", default=False,
 886                       help="Spew out even more information than normal")
 887     parser.add_option("-q", "--quiet", action="store_true", default=False,
 888                       help="Restrict output to warnings and errors")
 889     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 890                       help="Report on build data status")
 891     parser.add_option("-i", "--interactive", default=False, action="store_true",
 892                       help="Interactively ask about things that need updating.")
 893     parser.add_option("-I", "--icons", action="store_true", default=False,
 894                       help="Resize all the icons exceeding the max pixel size and exit")
 895     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 896                       help="Specify editor to use in interactive mode. Default " +
 897                           "is /etc/alternatives/editor")
 898     parser.add_option("-w", "--wiki", default=False, action="store_true",
 899                       help="Update the wiki")
 900     parser.add_option("", "--pretty", action="store_true", default=False,
 901                       help="Produce human-readable index.xml")
 902     parser.add_option("--clean", action="store_true", default=False,
 903                       help="Clean update - don't uses caches, reprocess all apks")
 904     (options, args) = parser.parse_args()
 905
 906     config = common.read_config(options)
 907
 908     repodirs = ['repo']
 909     if config['archive_older'] != 0:
 910         repodirs.append('archive')
 911         if not os.path.exists('archive'):
 912             os.mkdir('archive')
 913
 914     if options.icons:
 915         resize_all_icons(repodirs)
 916         sys.exit(0)
 917
 918     # Get all apps...
 919     apps = metadata.read_metadata()
 920
 921     # Generate a list of categories...
 922     categories = set()
 923     for app in apps:
 924         categories.update(app['Categories'])
 925
 926     # Read known apks data (will be updated and written back when we've finished)
 927     knownapks = common.KnownApks()
 928
 929     # Gather information about all the apk files in the repo directory, using
 930     # cached data if possible.
 931     apkcachefile = os.path.join('tmp', 'apkcache')
 932     if not options.clean and os.path.exists(apkcachefile):
 933         with open(apkcachefile, 'rb') as cf:
 934             apkcache = pickle.load(cf)
 935     else:
 936         apkcache = {}
 937     cachechanged = False
 938
 939     delete_disabled_builds(apps, apkcache, repodirs)
 940
 941     # Scan all apks in the main repo
 942     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 943     if cc:
 944         cachechanged = True
 945
 946     # Scan the archive repo for apks as well
 947     if len(repodirs) > 1:
 948         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 949         if cc:
 950             cachechanged = True
 951     else:
 952         archapks = []
 953
 954     # Some information from the apks needs to be applied up to the application
 955     # level. When doing this, we use the info from the most recent version's apk.
 956     # We deal with figuring out when the app was added and last updated at the
 957     # same time.
 958     for app in apps:
 959         bestver = 0
 960         added = None
 961         lastupdated = None
 962         for apk in apks + archapks:
 963             if apk['id'] == app['id']:
 964                 if apk['versioncode'] > bestver:
 965                     bestver = apk['versioncode']
 966                     bestapk = apk
 967
 968                 if 'added' in apk:
 969                     if not added or apk['added'] < added:
 970                         added = apk['added']
 971                     if not lastupdated or apk['added'] > lastupdated:
 972                         lastupdated = apk['added']
 973
 974         if added:
 975             app['added'] = added
 976         else:
 977             logging.warn("Don't know when " + app['id'] + " was added")
 978         if lastupdated:
 979             app['lastupdated'] = lastupdated
 980         else:
 981             logging.warn("Don't know when " + app['id'] + " was last updated")
 982
 983         if bestver == 0:
 984             if app['Name'] is None:
 985                 app['Name'] = app['id']
 986             app['icon'] = None
 987             logging.warn("Application " + app['id'] + " has no packages")
 988         else:
 989             if app['Name'] is None:
 990                 app['Name'] = bestapk['name']
 991             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 992
 993     # Sort the app list by name, then the web site doesn't have to by default.
 994     # (we had to wait until we'd scanned the apks to do this, because mostly the
 995     # name comes from there!)
 996     apps = sorted(apps, key=lambda app: app['Name'].upper())
 997
 998     # Generate warnings for apk's with no metadata (or create skeleton
 999     # metadata files, if requested on the command line)
1000     for apk in apks:
1001         found = False
1002         for app in apps:
1003             if app['id'] == apk['id']:
1004                 found = True
1005                 break
1006         if not found:
1007             if options.createmeta:
1008                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1009                 f.write("License:Unknown\n")
1010                 f.write("Web Site:\n")
1011                 f.write("Source Code:\n")
1012                 f.write("Issue Tracker:\n")
1013                 f.write("Summary:" + apk['name'] + "\n")
1014                 f.write("Description:\n")
1015                 f.write(apk['name'] + "\n")
1016                 f.write(".\n")
1017                 f.close()
1018                 logging.info("Generated skeleton metadata for " + apk['id'])
1019             else:
1020                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata - removing")
1021                 rmf = os.path.join(repodirs[0], apk['apkname'])
1022                 if not os.path.exists(rmf):
1023                     logging.error("Could not find {0} to remove it".format(rmf))
1024                 else:
1025                     os.remove(rmf)
1026
1027     if len(repodirs) > 1:
1028         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1029
1030     # Make the index for the main repo...
1031     make_index(apps, apks, repodirs[0], False, categories)
1032
1033     # If there's an archive repo,  make the index for it. We already scanned it
1034     # earlier on.
1035     if len(repodirs) > 1:
1036         make_index(apps, archapks, repodirs[1], True, categories)
1037
1038     if config['update_stats']:
1039
1040         # Update known apks info...
1041         knownapks.writeifchanged()
1042
1043         # Generate latest apps data for widget
1044         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1045             data = ''
1046             for line in file(os.path.join('stats', 'latestapps.txt')):
1047                 appid = line.rstrip()
1048                 data += appid + "\t"
1049                 for app in apps:
1050                     if app['id'] == appid:
1051                         data += app['Name'] + "\t"
1052                         if app['icon'] is not None:
1053                             data += app['icon'] + "\t"
1054                         data += app['License'] + "\n"
1055                         break
1056             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1057             f.write(data)
1058             f.close()
1059
1060     if cachechanged:
1061         with open(apkcachefile, 'wb') as cf:
1062             pickle.dump(apkcache, cf)
1063
1064     # Update the wiki...
1065     if options.wiki:
1066         update_wiki(apps, apks + archapks)
1067
1068     logging.info("Finished.")
1069
1070 if __name__ == "__main__":
1071     main()