fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common, metadata
  36 from common import FDroidPopen
  37 from metadata import MetaDataException
  38
  39 def get_densities():
  40     return ['640', '480', '320', '240', '160', '120']
  41
  42 def dpi_to_px(density):
  43     return (int(density) * 48) / 160
  44
  45 def px_to_dpi(px):
  46     return (int(px) * 160) / 48
  47
  48 def get_icon_dir(repodir, density):
  49     if density is None:
  50         return os.path.join(repodir, "icons")
  51     return os.path.join(repodir, "icons-%s" % density)
  52
  53 def get_icon_dirs(repodir):
  54     for density in get_densities():
  55         yield get_icon_dir(repodir, density)
  56     yield os.path.join(repodir, "icons")
  57
  58 def update_wiki(apps, apks):
  59     """Update the wiki
  60
  61     :param apps: fully populated list of all applications
  62     :param apks: all apks, except...
  63     """
  64     logging.info("Updating wiki")
  65     wikicat = 'Apps'
  66     wikiredircat = 'App Redirects'
  67     import mwclient
  68     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  69             path=config['wiki_path'])
  70     site.login(config['wiki_user'], config['wiki_password'])
  71     generated_pages = {}
  72     generated_redirects = {}
  73     for app in apps:
  74         wikidata = ''
  75         if app['Disabled']:
  76             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  77         if app['AntiFeatures']:
  78             for af in app['AntiFeatures'].split(','):
  79                 wikidata += '{{AntiFeature|' + af + '}}\n'
  80         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n'%(
  81                 app['id'],
  82                 app['Name'],
  83                 time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  84                 time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  85                 app['Source Code'],
  86                 app['Issue Tracker'],
  87                 app['Web Site'],
  88                 app['Donate'],
  89                 app['FlattrID'],
  90                 app['Bitcoin'],
  91                 app['Litecoin'],
  92                 app['Dogecoin'],
  93                 app['License'],
  94                 app.get('Requires Root', 'No'))
  95
  96         if app['Provides']:
  97             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
  98
  99         wikidata += app['Summary']
 100         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 101
 102         wikidata += "=Description=\n"
 103         wikidata += metadata.description_wiki(app['Description']) + "\n"
 104
 105         wikidata += "=Maintainer Notes=\n"
 106         if 'Maintainer Notes' in app:
 107             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 108         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 109
 110         # Get a list of all packages for this application...
 111         apklist = []
 112         gotcurrentver = False
 113         cantupdate = False
 114         buildfails = False
 115         for apk in apks:
 116             if apk['id'] == app['id']:
 117                 if str(apk['versioncode']) == app['Current Version Code']:
 118                     gotcurrentver = True
 119                 apklist.append(apk)
 120         # Include ones we can't build, as a special case...
 121         for thisbuild in app['builds']:
 122             if 'disable' in thisbuild:
 123                 if thisbuild['vercode'] == app['Current Version Code']:
 124                     cantupdate = True
 125                 apklist.append({
 126                         #TODO: Nasty: vercode is a string in the build, and an int elsewhere
 127                         'versioncode': int(thisbuild['vercode']),
 128                         'version': thisbuild['version'],
 129                         'buildproblem': thisbuild['disable']
 130                     })
 131             else:
 132                 builtit = False
 133                 for apk in apklist:
 134                     if apk['versioncode'] == int(thisbuild['vercode']):
 135                         builtit = True
 136                         break
 137                 if not builtit:
 138                     buildfails = True
 139                     apklist.append({
 140                             'versioncode': int(thisbuild['vercode']),
 141                             'version': thisbuild['version'],
 142                             'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 143                         })
 144         if app['Current Version Code'] == '0':
 145             cantupdate = True
 146         # Sort with most recent first...
 147         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 148
 149         wikidata += "=Versions=\n"
 150         if len(apklist) == 0:
 151             wikidata += "We currently have no versions of this app available."
 152         elif not gotcurrentver:
 153             wikidata += "We don't have the current version of this app."
 154         else:
 155             wikidata += "We have the current version of this app."
 156         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 157         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 158         if len(app['No Source Since']) > 0:
 159             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 160         if len(app['Current Version']) > 0:
 161             wikidata += "The current (recommended) version is " + app['Current Version']
 162             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 163         validapks = 0
 164         for apk in apklist:
 165             wikidata += "==" + apk['version'] + "==\n"
 166
 167             if 'buildproblem' in apk:
 168                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 169             else:
 170                 validapks += 1
 171                 wikidata += "This version is built and signed by "
 172                 if 'srcname' in apk:
 173                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 174                 else:
 175                     wikidata += "the original developer.\n\n"
 176             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 177
 178         wikidata += '\n[[Category:' + wikicat + ']]\n'
 179         if len(app['No Source Since']) > 0:
 180             wikidata += '\n[[Category:Apps missing source code]]\n'
 181         if validapks == 0 and not app['Disabled']:
 182             wikidata += '\n[[Category:Apps with no packages]]\n'
 183         if cantupdate and not app['Disabled']:
 184             wikidata += "\n[[Category:Apps we can't update]]\n"
 185         if buildfails and not app['Disabled']:
 186             wikidata += "\n[[Category:Apps with failing builds]]\n"
 187         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 188             wikidata += '\n[[Category:Apps to Update]]\n'
 189         if app['Disabled']:
 190             wikidata += '\n[[Category:Apps that are disabled]]\n'
 191         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 192             wikidata += '\n[[Category:Apps with no update check]]\n'
 193         for appcat in app['Categories']:
 194             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 195
 196         # We can't have underscores in the page name, even if they're in
 197         # the package ID, because MediaWiki messes with them...
 198         pagename = app['id'].replace('_', ' ')
 199
 200         # Drop a trailing newline, because mediawiki is going to drop it anyway
 201         # and it we don't we'll think the page has changed when it hasn't...
 202         if wikidata.endswith('\n'):
 203             wikidata = wikidata[:-1]
 204
 205         generated_pages[pagename] = wikidata
 206
 207         # Make a redirect from the name to the ID too, unless there's
 208         # already an existing page with the name and it isn't a redirect.
 209         noclobber = False
 210         apppagename = app['Name'].replace('_', ' ')
 211         apppagename = apppagename.replace('{', '')
 212         apppagename = apppagename.replace('}', ' ')
 213         apppagename = apppagename.replace(':', ' ')
 214         # Drop double spaces caused mostly by replacing ':' above
 215         apppagename = apppagename.replace('  ', ' ')
 216         for expagename in site.allpages(prefix=apppagename,
 217                 filterredir='nonredirects', generator=False):
 218             if expagename == apppagename:
 219                 noclobber = True
 220         # Another reason not to make the redirect page is if the app name
 221         # is the same as it's ID, because that will overwrite the real page
 222         # with an redirect to itself! (Although it seems like an odd
 223         # scenario this happens a lot, e.g. where there is metadata but no
 224         # builds or binaries to extract a name from.
 225         if apppagename == pagename:
 226             noclobber = True
 227         if not noclobber:
 228             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 229
 230     for tcat, genp in [(wikicat, generated_pages),
 231             (wikiredircat, generated_redirects)]:
 232         catpages = site.Pages['Category:' + tcat]
 233         existingpages = []
 234         for page in catpages:
 235             existingpages.append(page.name)
 236             if page.name in genp:
 237                 pagetxt = page.edit()
 238                 if pagetxt != genp[page.name]:
 239                     logging.debug("Updating modified page " + page.name)
 240                     page.save(genp[page.name], summary='Auto-updated')
 241                 else:
 242                     logging.debug("Page " + page.name + " is unchanged")
 243             else:
 244                 logging.warn("Deleting page " + page.name)
 245                 page.delete('No longer published')
 246         for pagename, text in genp.items():
 247             logging.debug("Checking " + pagename)
 248             if not pagename in existingpages:
 249                 logging.debug("Creating page " + pagename)
 250                 try:
 251                     newpage = site.Pages[pagename]
 252                     newpage.save(text, summary='Auto-created')
 253                 except:
 254                     logging.error("...FAILED to create page")
 255
 256     # Purge server cache to ensure counts are up to date
 257     site.pages['Repository Maintenance'].purge()
 258
 259 def delete_disabled_builds(apps, apkcache, repodirs):
 260     """Delete disabled build outputs.
 261
 262     :param apps: list of all applications, as per metadata.read_metadata
 263     :param apkcache: current apk cache information
 264     :param repodirs: the repo directories to process
 265     """
 266     for app in apps:
 267         for build in app['builds']:
 268             if 'disable' in build:
 269                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 270                 for repodir in repodirs:
 271                     apkpath = os.path.join(repodir, apkfilename)
 272                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 273                     for name in [apkpath, srcpath]:
 274                         if os.path.exists(name):
 275                             logging.warn("Deleting disabled build output " + apkfilename)
 276                             os.remove(name)
 277                 if apkfilename in apkcache:
 278                     del apkcache[apkfilename]
 279
 280 def resize_icon(iconpath, density):
 281
 282     if not os.path.isfile(iconpath):
 283         return
 284
 285     try:
 286         im = Image.open(iconpath)
 287         size = dpi_to_px(density)
 288
 289         if any(length > size for length in im.size):
 290             oldsize = im.size
 291             im.thumbnail((size, size), Image.ANTIALIAS)
 292             logging.debug("%s was too large at %s - new size is %s" % (
 293                 iconpath, oldsize, im.size))
 294             im.save(iconpath, "PNG")
 295
 296         else:
 297             logging.info("%s is small enough: %s" % im.size)
 298
 299     except Exception,e:
 300         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 301
 302 def resize_all_icons(repodirs):
 303     """Resize all icons that exceed the max size
 304
 305     :param repodirs: the repo directories to process
 306     """
 307     for repodir in repodirs:
 308         for density in get_densities():
 309             icon_dir = get_icon_dir(repodir, density)
 310             icon_glob = os.path.join(icon_dir, '*.png')
 311             for iconpath in glob.glob(icon_glob):
 312                 resize_icon(iconpath, density)
 313
 314 def scan_apks(apps, apkcache, repodir, knownapks):
 315     """Scan the apks in the given repo directory.
 316
 317     This also extracts the icons.
 318
 319     :param apps: list of all applications, as per metadata.read_metadata
 320     :param apkcache: current apk cache information
 321     :param repodir: repo directory to scan
 322     :param knownapks: known apks info
 323     :returns: (apks, cachechanged) where apks is a list of apk information,
 324               and cachechanged is True if the apkcache got changed.
 325     """
 326
 327     cachechanged = False
 328
 329     icon_dirs = get_icon_dirs(repodir)
 330     for icon_dir in icon_dirs:
 331         if os.path.exists(icon_dir):
 332             if options.clean:
 333                 shutil.rmtree(icon_dir)
 334                 os.makedirs(icon_dir)
 335         else:
 336             os.makedirs(icon_dir)
 337
 338     apks = []
 339     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 340     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 341     vername_pat = re.compile(".*versionName='([^']*)'.*")
 342     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 343     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 344     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 345     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 346     string_pat = re.compile(".*'([^']*)'.*")
 347     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 348
 349         apkfilename = apkfile[len(repodir) + 1:]
 350         if ' ' in apkfilename:
 351             logging.error("No spaces in APK filenames!")
 352             sys.exit(1)
 353
 354         if apkfilename in apkcache:
 355             logging.debug("Reading " + apkfilename + " from cache")
 356             thisinfo = apkcache[apkfilename]
 357
 358         else:
 359
 360             logging.info("Processing " + apkfilename)
 361             thisinfo = {}
 362             thisinfo['apkname'] = apkfilename
 363             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 364             if os.path.exists(os.path.join(repodir, srcfilename)):
 365                 thisinfo['srcname'] = srcfilename
 366             thisinfo['size'] = os.path.getsize(apkfile)
 367             thisinfo['permissions'] = []
 368             thisinfo['features'] = []
 369             thisinfo['icons_src'] = {}
 370             thisinfo['icons'] = {}
 371             p = FDroidPopen([os.path.join(config['sdk_path'],
 372                         'build-tools', config['build_tools'], 'aapt'),
 373                     'dump', 'badging', apkfile])
 374             if p.returncode != 0:
 375                 logging.critical("Failed to get apk information")
 376                 sys.exit(1)
 377             for line in p.stdout.splitlines():
 378                 if line.startswith("package:"):
 379                     try:
 380                         thisinfo['id'] = re.match(name_pat, line).group(1)
 381                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 382                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 383                     except Exception, e:
 384                         logging.info("Package matching failed: " + str(e))
 385                         logging.info("Line was: " + line)
 386                         sys.exit(1)
 387                 elif line.startswith("application:"):
 388                     thisinfo['name'] = re.match(label_pat, line).group(1)
 389                     # Keep path to non-dpi icon in case we need it
 390                     match = re.match(icon_pat_nodpi, line)
 391                     if match:
 392                         thisinfo['icons_src']['-1'] = match.group(1)
 393                 elif line.startswith("launchable-activity:"):
 394                     # Only use launchable-activity as fallback to application
 395                     if '-1' not in thisinfo['icons_src']:
 396                         match = re.match(icon_pat_nodpi, line)
 397                         if match:
 398                             thisinfo['icons_src']['-1'] = match.group(1)
 399                 elif line.startswith("application-icon-"):
 400                     match = re.match(icon_pat, line)
 401                     if match:
 402                         density = match.group(1)
 403                         path = match.group(2)
 404                         thisinfo['icons_src'][density] = path
 405                 elif line.startswith("sdkVersion:"):
 406                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 407                 elif line.startswith("maxSdkVersion:"):
 408                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 409                 elif line.startswith("native-code:"):
 410                     thisinfo['nativecode'] = []
 411                     for arch in line[13:].split(' '):
 412                         thisinfo['nativecode'].append(arch[1:-1])
 413                 elif line.startswith("uses-permission:"):
 414                     perm = re.match(string_pat, line).group(1)
 415                     if perm.startswith("android.permission."):
 416                         perm = perm[19:]
 417                     thisinfo['permissions'].append(perm)
 418                 elif line.startswith("uses-feature:"):
 419                     perm = re.match(string_pat, line).group(1)
 420                     #Filter out this, it's only added with the latest SDK tools and
 421                     #causes problems for lots of apps.
 422                     if (perm != "android.hardware.screen.portrait" and
 423                         perm != "android.hardware.screen.landscape"):
 424                         if perm.startswith("android.feature."):
 425                             perm = perm[16:]
 426                         thisinfo['features'].append(perm)
 427
 428             if not 'sdkversion' in thisinfo:
 429                 logging.warn("no SDK version information found")
 430                 thisinfo['sdkversion'] = 0
 431
 432             # Check for debuggable apks...
 433             if common.isApkDebuggable(apkfile, config):
 434                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 435
 436             # Calculate the sha256...
 437             sha = hashlib.sha256()
 438             with open(apkfile, 'rb') as f:
 439                 while True:
 440                     t = f.read(1024)
 441                     if len(t) == 0:
 442                         break
 443                     sha.update(t)
 444                 thisinfo['sha256'] = sha.hexdigest()
 445
 446             # Get the signature (or md5 of, to be precise)...
 447             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 448             if not os.path.exists(getsig_dir + "/getsig.class"):
 449                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 450                 sys.exit(1)
 451             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 452                         'getsig', os.path.join(os.getcwd(), apkfile)])
 453             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 454                 logging.critical("Failed to get apk signature")
 455                 sys.exit(1)
 456             thisinfo['sig'] = p.stdout[7:].strip()
 457
 458             apk = zipfile.ZipFile(apkfile, 'r')
 459
 460             iconfilename = "%s.%s.png" % (
 461                     thisinfo['id'],
 462                     thisinfo['versioncode'])
 463
 464             # Extract the icon file...
 465             densities = get_densities()
 466             empty_densities = []
 467             for density in densities:
 468                 if density not in thisinfo['icons_src']:
 469                     empty_densities.append(density)
 470                     continue
 471                 iconsrc = thisinfo['icons_src'][density]
 472                 icon_dir = get_icon_dir(repodir, density)
 473                 icondest = os.path.join(icon_dir, iconfilename)
 474
 475                 try:
 476                     iconfile = open(icondest, 'wb')
 477                     iconfile.write(apk.read(iconsrc))
 478                     iconfile.close()
 479                     thisinfo['icons'][density] = iconfilename
 480
 481                 except:
 482                     logging.warn("Error retrieving icon file")
 483                     del thisinfo['icons'][density]
 484                     del thisinfo['icons_src'][density]
 485                     empty_densities.append(density)
 486
 487             if '-1' in thisinfo['icons_src']:
 488                 iconsrc = thisinfo['icons_src']['-1']
 489                 iconpath = os.path.join(
 490                         get_icon_dir(repodir, None), iconfilename)
 491                 iconfile = open(iconpath, 'wb')
 492                 iconfile.write(apk.read(iconsrc))
 493                 iconfile.close()
 494                 try:
 495                     im = Image.open(iconpath)
 496                     dpi = px_to_dpi(im.size[0])
 497                     for density in densities:
 498                         if density in thisinfo['icons']:
 499                             break
 500                         if density == densities[-1] or dpi >= int(density):
 501                             thisinfo['icons'][density] = iconfilename
 502                             shutil.move(iconpath,
 503                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 504                             empty_densities.remove(density)
 505                             break
 506                 except Exception,e:
 507                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 508
 509             if thisinfo['icons']:
 510                 thisinfo['icon'] = iconfilename
 511
 512             apk.close()
 513
 514             # First try resizing down to not lose quality
 515             last_density = None
 516             for density in densities:
 517                 if density not in empty_densities:
 518                     last_density = density
 519                     continue
 520                 if last_density is None:
 521                     continue
 522                 logging.info("Density %s not available, resizing down from %s" % (
 523                         density, last_density))
 524
 525                 last_iconpath = os.path.join(
 526                         get_icon_dir(repodir, last_density), iconfilename)
 527                 iconpath = os.path.join(
 528                         get_icon_dir(repodir, density), iconfilename)
 529                 try:
 530                     im = Image.open(last_iconpath)
 531                 except:
 532                     logging.warn("Invalid image file at %s" % last_iconpath)
 533                     continue
 534
 535                 size = dpi_to_px(density)
 536
 537                 im.thumbnail((size, size), Image.ANTIALIAS)
 538                 im.save(iconpath, "PNG")
 539                 empty_densities.remove(density)
 540
 541             # Then just copy from the highest resolution available
 542             last_density = None
 543             for density in reversed(densities):
 544                 if density not in empty_densities:
 545                     last_density = density
 546                     continue
 547                 if last_density is None:
 548                     continue
 549                 logging.info("Density %s not available, copying from lower density %s" % (
 550                         density, last_density))
 551
 552                 shutil.copyfile(
 553                         os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 554                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 555
 556                 empty_densities.remove(density)
 557
 558             for density in densities:
 559                 icon_dir = get_icon_dir(repodir, density)
 560                 icondest = os.path.join(icon_dir, iconfilename)
 561                 resize_icon(icondest, density)
 562
 563             # Copy from icons-mdpi to icons since mdpi is the baseline density
 564             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 565             if os.path.isfile(baseline):
 566                 shutil.copyfile(baseline,
 567                         os.path.join(get_icon_dir(repodir, None), iconfilename))
 568
 569             # Record in known apks, getting the added date at the same time..
 570             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 571             if added:
 572                 thisinfo['added'] = added
 573
 574             apkcache[apkfilename] = thisinfo
 575             cachechanged = True
 576
 577         apks.append(thisinfo)
 578
 579     return apks, cachechanged
 580
 581
 582 repo_pubkey_fingerprint = None
 583
 584 def make_index(apps, apks, repodir, archive, categories):
 585     """Make a repo index.
 586
 587     :param apps: fully populated apps list
 588     :param apks: full populated apks list
 589     :param repodir: the repo directory
 590     :param archive: True if this is the archive repo, False if it's the
 591                     main one.
 592     :param categories: list of categories
 593     """
 594
 595     doc = Document()
 596
 597     def addElement(name, value, doc, parent):
 598         el = doc.createElement(name)
 599         el.appendChild(doc.createTextNode(value))
 600         parent.appendChild(el)
 601     def addElementCDATA(name, value, doc, parent):
 602         el = doc.createElement(name)
 603         el.appendChild(doc.createCDATASection(value))
 604         parent.appendChild(el)
 605
 606     root = doc.createElement("fdroid")
 607     doc.appendChild(root)
 608
 609     repoel = doc.createElement("repo")
 610
 611     if archive:
 612         repoel.setAttribute("name", config['archive_name'])
 613         if config['repo_maxage'] != 0:
 614             repoel.setAttribute("maxage", str(config['repo_maxage']))
 615         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 616         repoel.setAttribute("url", config['archive_url'])
 617         addElement('description', config['archive_description'], doc, repoel)
 618
 619     else:
 620         repoel.setAttribute("name", config['repo_name'])
 621         if config['repo_maxage'] != 0:
 622             repoel.setAttribute("maxage", str(config['repo_maxage']))
 623         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 624         repoel.setAttribute("url", config['repo_url'])
 625         addElement('description', config['repo_description'], doc, repoel)
 626
 627     repoel.setAttribute("version", "12")
 628     repoel.setAttribute("timestamp", str(int(time.time())))
 629
 630     if 'repo_keyalias' in config:
 631
 632         # Generate a certificate fingerprint the same way keytool does it
 633         # (but with slightly different formatting)
 634         def cert_fingerprint(data):
 635             digest = hashlib.sha1(data).digest()
 636             ret = []
 637             for i in range(4):
 638                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 639             return " ".join(ret)
 640
 641         def extract_pubkey():
 642             p = FDroidPopen(['keytool', '-exportcert',
 643                                   '-alias', config['repo_keyalias'],
 644                                   '-keystore', config['keystore'],
 645                                   '-storepass:file', config['keystorepassfile']]
 646                             + config['smartcardoptions'])
 647             if p.returncode != 0:
 648                 logging.critical("Failed to get repo pubkey")
 649                 sys.exit(1)
 650             global repo_pubkey_fingerprint
 651             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 652             return "".join("%02x" % ord(b) for b in p.stdout)
 653
 654         repoel.setAttribute("pubkey", extract_pubkey())
 655
 656     root.appendChild(repoel)
 657
 658     for app in apps:
 659
 660         if app['Disabled'] is not None:
 661             continue
 662
 663         # Get a list of the apks for this app...
 664         apklist = []
 665         for apk in apks:
 666             if apk['id'] == app['id']:
 667                 apklist.append(apk)
 668
 669         if len(apklist) == 0:
 670             continue
 671
 672         apel = doc.createElement("application")
 673         apel.setAttribute("id", app['id'])
 674         root.appendChild(apel)
 675
 676         addElement('id', app['id'], doc, apel)
 677         if 'added' in app:
 678             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 679         if 'lastupdated' in app:
 680             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 681         addElement('name', app['Name'], doc, apel)
 682         addElement('summary', app['Summary'], doc, apel)
 683         if app['icon']:
 684             addElement('icon', app['icon'], doc, apel)
 685         def linkres(link):
 686             for app in apps:
 687                 if app['id'] == link:
 688                     return ("fdroid.app:" + link, app['Name'])
 689             raise MetaDataException("Cannot resolve app id " + link)
 690         addElement('desc',
 691                 metadata.description_html(app['Description'], linkres), doc, apel)
 692         addElement('license', app['License'], doc, apel)
 693         if 'Categories' in app:
 694             addElement('categories', ','.join(app["Categories"]), doc, apel)
 695             # We put the first (primary) category in LAST, which will have
 696             # the desired effect of making clients that only understand one
 697             # category see that one.
 698             addElement('category', app["Categories"][0], doc, apel)
 699         addElement('web', app['Web Site'], doc, apel)
 700         addElement('source', app['Source Code'], doc, apel)
 701         addElement('tracker', app['Issue Tracker'], doc, apel)
 702         if app['Donate']:
 703             addElement('donate', app['Donate'], doc, apel)
 704         if app['Bitcoin']:
 705             addElement('bitcoin', app['Bitcoin'], doc, apel)
 706         if app['Litecoin']:
 707             addElement('litecoin', app['Litecoin'], doc, apel)
 708         if app['Dogecoin']:
 709             addElement('dogecoin', app['Dogecoin'], doc, apel)
 710         if app['FlattrID']:
 711             addElement('flattr', app['FlattrID'], doc, apel)
 712
 713         # These elements actually refer to the current version (i.e. which
 714         # one is recommended. They are historically mis-named, and need
 715         # changing, but stay like this for now to support existing clients.
 716         addElement('marketversion', app['Current Version'], doc, apel)
 717         addElement('marketvercode', app['Current Version Code'], doc, apel)
 718
 719         if app['AntiFeatures']:
 720             af = app['AntiFeatures'].split(',')
 721             # TODO: Temporarily not including UpstreamNonFree in the index,
 722             # because current F-Droid clients do not understand it, and also
 723             # look ugly when they encounter an unknown antifeature. This
 724             # filtering can be removed in time...
 725             if 'UpstreamNonFree' in af:
 726                 af.remove('UpstreamNonFree')
 727             if af:
 728                 addElement('antifeatures', ','.join(af), doc, apel)
 729         if app['Provides']:
 730             pv = app['Provides'].split(',')
 731             addElement('provides', ','.join(pv), doc, apel)
 732         if app['Requires Root']:
 733             addElement('requirements', 'root', doc, apel)
 734
 735         # Sort the apk list into version order, just so the web site
 736         # doesn't have to do any work by default...
 737         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 738
 739         # Check for duplicates - they will make the client unhappy...
 740         for i in range(len(apklist) - 1):
 741             if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 742                 logging.critical("duplicate versions: '%s' - '%s'" % (
 743                     apklist[i]['apkname'], apklist[i+1]['apkname']))
 744                 sys.exit(1)
 745
 746         for apk in apklist:
 747             apkel = doc.createElement("package")
 748             apel.appendChild(apkel)
 749             addElement('version', apk['version'], doc, apkel)
 750             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 751             addElement('apkname', apk['apkname'], doc, apkel)
 752             if 'srcname' in apk:
 753                 addElement('srcname', apk['srcname'], doc, apkel)
 754             for hash_type in ['sha256']:
 755                 if not hash_type in apk:
 756                     continue
 757                 hashel = doc.createElement("hash")
 758                 hashel.setAttribute("type", hash_type)
 759                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 760                 apkel.appendChild(hashel)
 761             addElement('sig', apk['sig'], doc, apkel)
 762             addElement('size', str(apk['size']), doc, apkel)
 763             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 764             if 'maxsdkversion' in apk:
 765                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 766             if 'added' in apk:
 767                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 768             if app['Requires Root']:
 769                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 770                     apk['permissions'].append('ACCESS_SUPERUSER')
 771
 772             if len(apk['permissions']) > 0:
 773                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 774             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 775                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 776             if len(apk['features']) > 0:
 777                 addElement('features', ','.join(apk['features']), doc, apkel)
 778
 779     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 780     if options.pretty:
 781         output = doc.toprettyxml()
 782     else:
 783         output = doc.toxml()
 784     of.write(output)
 785     of.close()
 786
 787     if 'repo_keyalias' in config:
 788
 789         logging.info("Creating signed index.")
 790         logging.info("Key fingerprint: %s" % repo_pubkey_fingerprint)
 791
 792         #Create a jar of the index...
 793         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 794         if p.returncode != 0:
 795             logging.critical("Failed to create jar file")
 796             sys.exit(1)
 797
 798         # Sign the index...
 799         args = ['jarsigner', '-keystore', config['keystore'],
 800                 '-storepass:file', config['keystorepassfile'],
 801                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 802                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 803         if config['keystore'] == 'NONE':
 804             args += config['smartcardoptions']
 805         else:  # smardcards never use -keypass
 806             args += ['-keypass:file', config['keypassfile']]
 807         p = FDroidPopen(args)
 808         # TODO keypass should be sent via stdin
 809         if p.returncode != 0:
 810             logging.info("Failed to sign index")
 811             sys.exit(1)
 812
 813     # Copy the repo icon into the repo directory...
 814     icon_dir = os.path.join(repodir ,'icons')
 815     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 816     shutil.copyfile(config['repo_icon'], iconfilename)
 817
 818     # Write a category list in the repo to allow quick access...
 819     catdata = ''
 820     for cat in categories:
 821         catdata += cat + '\n'
 822     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 823     f.write(catdata)
 824     f.close()
 825
 826
 827
 828 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 829
 830     for app in apps:
 831
 832         # Get a list of the apks for this app...
 833         apklist = []
 834         for apk in apks:
 835             if apk['id'] == app['id']:
 836                 apklist.append(apk)
 837
 838         # Sort the apk list into version order...
 839         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 840
 841         if app['Archive Policy']:
 842             keepversions = int(app['Archive Policy'][:-9])
 843         else:
 844             keepversions = defaultkeepversions
 845
 846         if len(apklist) > keepversions:
 847             for apk in apklist[keepversions:]:
 848                 logging.info("Moving " + apk['apkname'] + " to archive")
 849                 shutil.move(os.path.join(repodir, apk['apkname']),
 850                     os.path.join(archivedir, apk['apkname']))
 851                 if 'srcname' in apk:
 852                     shutil.move(os.path.join(repodir, apk['srcname']),
 853                         os.path.join(archivedir, apk['srcname']))
 854                 archapks.append(apk)
 855                 apks.remove(apk)
 856
 857
 858 config = None
 859 options = None
 860
 861 def main():
 862
 863     global config, options
 864
 865     # Parse command line...
 866     parser = OptionParser()
 867     parser.add_option("-c", "--createmeta", action="store_true", default=False,
 868                       help="Create skeleton metadata files that are missing")
 869     parser.add_option("-v", "--verbose", action="store_true", default=False,
 870                       help="Spew out even more information than normal")
 871     parser.add_option("-q", "--quiet", action="store_true", default=False,
 872                       help="Restrict output to warnings and errors")
 873     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 874                       help="Report on build data status")
 875     parser.add_option("-i", "--interactive", default=False, action="store_true",
 876                       help="Interactively ask about things that need updating.")
 877     parser.add_option("-I", "--icons", action="store_true", default=False,
 878                       help="Resize all the icons exceeding the max pixel size and exit")
 879     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 880                       help="Specify editor to use in interactive mode. Default "+
 881                           "is /etc/alternatives/editor")
 882     parser.add_option("-w", "--wiki", default=False, action="store_true",
 883                       help="Update the wiki")
 884     parser.add_option("", "--pretty", action="store_true", default=False,
 885                       help="Produce human-readable index.xml")
 886     parser.add_option("--clean", action="store_true", default=False,
 887                       help="Clean update - don't uses caches, reprocess all apks")
 888     (options, args) = parser.parse_args()
 889
 890     config = common.read_config(options)
 891
 892     repodirs = ['repo']
 893     if config['archive_older'] != 0:
 894         repodirs.append('archive')
 895         if not os.path.exists('archive'):
 896             os.mkdir('archive')
 897
 898     if options.icons:
 899         resize_all_icons(repodirs)
 900         sys.exit(0)
 901
 902     # Get all apps...
 903     apps = metadata.read_metadata()
 904
 905     # Generate a list of categories...
 906     categories = set()
 907     for app in apps:
 908         categories.update(app['Categories'])
 909
 910     # Read known apks data (will be updated and written back when we've finished)
 911     knownapks = common.KnownApks()
 912
 913     # Gather information about all the apk files in the repo directory, using
 914     # cached data if possible.
 915     apkcachefile = os.path.join('tmp', 'apkcache')
 916     if not options.clean and os.path.exists(apkcachefile):
 917         with open(apkcachefile, 'rb') as cf:
 918             apkcache = pickle.load(cf)
 919     else:
 920         apkcache = {}
 921     cachechanged = False
 922
 923     delete_disabled_builds(apps, apkcache, repodirs)
 924
 925     # Scan all apks in the main repo
 926     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 927     if cc:
 928         cachechanged = True
 929
 930     # Scan the archive repo for apks as well
 931     if len(repodirs) > 1:
 932         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 933         if cc:
 934             cachechanged = True
 935     else:
 936         archapks = []
 937
 938     # Some information from the apks needs to be applied up to the application
 939     # level. When doing this, we use the info from the most recent version's apk.
 940     # We deal with figuring out when the app was added and last updated at the
 941     # same time.
 942     for app in apps:
 943         bestver = 0
 944         added = None
 945         lastupdated = None
 946         for apk in apks + archapks:
 947             if apk['id'] == app['id']:
 948                 if apk['versioncode'] > bestver:
 949                     bestver = apk['versioncode']
 950                     bestapk = apk
 951
 952                 if 'added' in apk:
 953                     if not added or apk['added'] < added:
 954                         added = apk['added']
 955                     if not lastupdated or apk['added'] > lastupdated:
 956                         lastupdated = apk['added']
 957
 958         if added:
 959             app['added'] = added
 960         else:
 961             logging.warn("Don't know when " + app['id'] + " was added")
 962         if lastupdated:
 963             app['lastupdated'] = lastupdated
 964         else:
 965             logging.warn("Don't know when " + app['id'] + " was last updated")
 966
 967         if bestver == 0:
 968             if app['Name'] is None:
 969                 app['Name'] = app['id']
 970             app['icon'] = None
 971             logging.warn("Application " + app['id'] + " has no packages")
 972         else:
 973             if app['Name'] is None:
 974                 app['Name'] = bestapk['name']
 975             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
 976
 977     # Sort the app list by name, then the web site doesn't have to by default.
 978     # (we had to wait until we'd scanned the apks to do this, because mostly the
 979     # name comes from there!)
 980     apps = sorted(apps, key=lambda app: app['Name'].upper())
 981
 982     # Generate warnings for apk's with no metadata (or create skeleton
 983     # metadata files, if requested on the command line)
 984     for apk in apks:
 985         found = False
 986         for app in apps:
 987             if app['id'] == apk['id']:
 988                 found = True
 989                 break
 990         if not found:
 991             if options.createmeta:
 992                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 993                 f.write("License:Unknown\n")
 994                 f.write("Web Site:\n")
 995                 f.write("Source Code:\n")
 996                 f.write("Issue Tracker:\n")
 997                 f.write("Summary:" + apk['name'] + "\n")
 998                 f.write("Description:\n")
 999                 f.write(apk['name'] + "\n")
1000                 f.write(".\n")
1001                 f.close()
1002                 logging.info("Generated skeleton metadata for " + apk['id'])
1003             else:
1004                 logging.warn(apk['apkname'] + " (" + apk['id'] + ") has no metadata - removing")
1005                 rmf = os.path.join(repodirs[0], apk['apkname'])
1006                 if not os.path.exists(rmf):
1007                     logging.error("Could not find {0} to remove it".format(rmf))
1008                 else:
1009                     os.remove(rmf)
1010
1011     if len(repodirs) > 1:
1012         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1013
1014     # Make the index for the main repo...
1015     make_index(apps, apks, repodirs[0], False, categories)
1016
1017     # If there's an archive repo,  make the index for it. We already scanned it
1018     # earlier on.
1019     if len(repodirs) > 1:
1020         make_index(apps, archapks, repodirs[1], True, categories)
1021
1022     if config['update_stats']:
1023
1024         # Update known apks info...
1025         knownapks.writeifchanged()
1026
1027         # Generate latest apps data for widget
1028         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1029             data = ''
1030             for line in file(os.path.join('stats', 'latestapps.txt')):
1031                 appid = line.rstrip()
1032                 data += appid + "\t"
1033                 for app in apps:
1034                     if app['id'] == appid:
1035                         data += app['Name'] + "\t"
1036                         if app['icon'] is not None:
1037                             data += app['icon'] + "\t"
1038                         data += app['License'] + "\n"
1039                         break
1040             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1041             f.write(data)
1042             f.close()
1043
1044     if cachechanged:
1045         with open(apkcachefile, 'wb') as cf:
1046             pickle.dump(apkcache, cf)
1047
1048     # Update the wiki...
1049     if options.wiki:
1050         update_wiki(apps, apks + archapks)
1051
1052     logging.info("Finished.")
1053
1054 if __name__ == "__main__":
1055     main()
1056