fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304         else:
 305             logging.info("%s is small enough: %s" % im.size)
 306
 307     except Exception, e:
 308         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 309
 310
 311 def resize_all_icons(repodirs):
 312     """Resize all icons that exceed the max size
 313
 314     :param repodirs: the repo directories to process
 315     """
 316     for repodir in repodirs:
 317         for density in get_densities():
 318             icon_dir = get_icon_dir(repodir, density)
 319             icon_glob = os.path.join(icon_dir, '*.png')
 320             for iconpath in glob.glob(icon_glob):
 321                 resize_icon(iconpath, density)
 322
 323
 324 def scan_apks(apps, apkcache, repodir, knownapks):
 325     """Scan the apks in the given repo directory.
 326
 327     This also extracts the icons.
 328
 329     :param apps: list of all applications, as per metadata.read_metadata
 330     :param apkcache: current apk cache information
 331     :param repodir: repo directory to scan
 332     :param knownapks: known apks info
 333     :returns: (apks, cachechanged) where apks is a list of apk information,
 334               and cachechanged is True if the apkcache got changed.
 335     """
 336
 337     cachechanged = False
 338
 339     icon_dirs = get_icon_dirs(repodir)
 340     for icon_dir in icon_dirs:
 341         if os.path.exists(icon_dir):
 342             if options.clean:
 343                 shutil.rmtree(icon_dir)
 344                 os.makedirs(icon_dir)
 345         else:
 346             os.makedirs(icon_dir)
 347
 348     apks = []
 349     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 350     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 351     vername_pat = re.compile(".*versionName='([^']*)'.*")
 352     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 353     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 354     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 355     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 356     string_pat = re.compile(".*'([^']*)'.*")
 357     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 358
 359         apkfilename = apkfile[len(repodir) + 1:]
 360         if ' ' in apkfilename:
 361             logging.error("No spaces in APK filenames!")
 362             sys.exit(1)
 363
 364         if apkfilename in apkcache:
 365             logging.debug("Reading " + apkfilename + " from cache")
 366             thisinfo = apkcache[apkfilename]
 367
 368         else:
 369
 370             logging.info("Processing " + apkfilename)
 371             thisinfo = {}
 372             thisinfo['apkname'] = apkfilename
 373             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 374             if os.path.exists(os.path.join(repodir, srcfilename)):
 375                 thisinfo['srcname'] = srcfilename
 376             thisinfo['size'] = os.path.getsize(apkfile)
 377             thisinfo['permissions'] = []
 378             thisinfo['features'] = []
 379             thisinfo['icons_src'] = {}
 380             thisinfo['icons'] = {}
 381             p = FDroidPopen([os.path.join(config['sdk_path'], 'build-tools',
 382                                           config['build_tools'], 'aapt'),
 383                              'dump', 'badging', apkfile])
 384             if p.returncode != 0:
 385                 logging.critical("Failed to get apk information")
 386                 sys.exit(1)
 387             for line in p.stdout.splitlines():
 388                 if line.startswith("package:"):
 389                     try:
 390                         thisinfo['id'] = re.match(name_pat, line).group(1)
 391                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 392                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 393                     except Exception, e:
 394                         logging.info("Package matching failed: " + str(e))
 395                         logging.info("Line was: " + line)
 396                         sys.exit(1)
 397                 elif line.startswith("application:"):
 398                     thisinfo['name'] = re.match(label_pat, line).group(1)
 399                     # Keep path to non-dpi icon in case we need it
 400                     match = re.match(icon_pat_nodpi, line)
 401                     if match:
 402                         thisinfo['icons_src']['-1'] = match.group(1)
 403                 elif line.startswith("launchable-activity:"):
 404                     # Only use launchable-activity as fallback to application
 405                     if not thisinfo['name']:
 406                         thisinfo['name'] = re.match(label_pat, line).group(1)
 407                     if '-1' not in thisinfo['icons_src']:
 408                         match = re.match(icon_pat_nodpi, line)
 409                         if match:
 410                             thisinfo['icons_src']['-1'] = match.group(1)
 411                 elif line.startswith("application-icon-"):
 412                     match = re.match(icon_pat, line)
 413                     if match:
 414                         density = match.group(1)
 415                         path = match.group(2)
 416                         thisinfo['icons_src'][density] = path
 417                 elif line.startswith("sdkVersion:"):
 418                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 419                 elif line.startswith("maxSdkVersion:"):
 420                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 421                 elif line.startswith("native-code:"):
 422                     thisinfo['nativecode'] = []
 423                     for arch in line[13:].split(' '):
 424                         thisinfo['nativecode'].append(arch[1:-1])
 425                 elif line.startswith("uses-permission:"):
 426                     perm = re.match(string_pat, line).group(1)
 427                     if perm.startswith("android.permission."):
 428                         perm = perm[19:]
 429                     thisinfo['permissions'].append(perm)
 430                 elif line.startswith("uses-feature:"):
 431                     perm = re.match(string_pat, line).group(1)
 432                     # Filter out this, it's only added with the latest SDK tools and
 433                     # causes problems for lots of apps.
 434                     if perm != "android.hardware.screen.portrait" \
 435                             and perm != "android.hardware.screen.landscape":
 436                         if perm.startswith("android.feature."):
 437                             perm = perm[16:]
 438                         thisinfo['features'].append(perm)
 439
 440             if 'sdkversion' not in thisinfo:
 441                 logging.warn("no SDK version information found")
 442                 thisinfo['sdkversion'] = 0
 443
 444             # Check for debuggable apks...
 445             if common.isApkDebuggable(apkfile, config):
 446                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 447
 448             # Calculate the sha256...
 449             sha = hashlib.sha256()
 450             with open(apkfile, 'rb') as f:
 451                 while True:
 452                     t = f.read(1024)
 453                     if len(t) == 0:
 454                         break
 455                     sha.update(t)
 456                 thisinfo['sha256'] = sha.hexdigest()
 457
 458             # Get the signature (or md5 of, to be precise)...
 459             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 460             if not os.path.exists(getsig_dir + "/getsig.class"):
 461                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 462                 sys.exit(1)
 463             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 464                              'getsig', os.path.join(os.getcwd(), apkfile)])
 465             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 466                 logging.critical("Failed to get apk signature")
 467                 sys.exit(1)
 468             thisinfo['sig'] = p.stdout[7:].strip()
 469
 470             apk = zipfile.ZipFile(apkfile, 'r')
 471
 472             iconfilename = "%s.%s.png" % (
 473                 thisinfo['id'],
 474                 thisinfo['versioncode'])
 475
 476             # Extract the icon file...
 477             densities = get_densities()
 478             empty_densities = []
 479             for density in densities:
 480                 if density not in thisinfo['icons_src']:
 481                     empty_densities.append(density)
 482                     continue
 483                 iconsrc = thisinfo['icons_src'][density]
 484                 icon_dir = get_icon_dir(repodir, density)
 485                 icondest = os.path.join(icon_dir, iconfilename)
 486
 487                 try:
 488                     iconfile = open(icondest, 'wb')
 489                     iconfile.write(apk.read(iconsrc))
 490                     iconfile.close()
 491                     thisinfo['icons'][density] = iconfilename
 492
 493                 except:
 494                     logging.warn("Error retrieving icon file")
 495                     del thisinfo['icons'][density]
 496                     del thisinfo['icons_src'][density]
 497                     empty_densities.append(density)
 498
 499             if '-1' in thisinfo['icons_src']:
 500                 iconsrc = thisinfo['icons_src']['-1']
 501                 iconpath = os.path.join(
 502                     get_icon_dir(repodir, None), iconfilename)
 503                 iconfile = open(iconpath, 'wb')
 504                 iconfile.write(apk.read(iconsrc))
 505                 iconfile.close()
 506                 try:
 507                     im = Image.open(iconpath)
 508                     dpi = px_to_dpi(im.size[0])
 509                     for density in densities:
 510                         if density in thisinfo['icons']:
 511                             break
 512                         if density == densities[-1] or dpi >= int(density):
 513                             thisinfo['icons'][density] = iconfilename
 514                             shutil.move(iconpath,
 515                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 516                             empty_densities.remove(density)
 517                             break
 518                 except Exception, e:
 519                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 520
 521             if thisinfo['icons']:
 522                 thisinfo['icon'] = iconfilename
 523
 524             apk.close()
 525
 526             # First try resizing down to not lose quality
 527             last_density = None
 528             for density in densities:
 529                 if density not in empty_densities:
 530                     last_density = density
 531                     continue
 532                 if last_density is None:
 533                     continue
 534                 logging.info("Density %s not available, resizing down from %s"
 535                              % (density, last_density))
 536
 537                 last_iconpath = os.path.join(
 538                     get_icon_dir(repodir, last_density), iconfilename)
 539                 iconpath = os.path.join(
 540                     get_icon_dir(repodir, density), iconfilename)
 541                 try:
 542                     im = Image.open(last_iconpath)
 543                 except:
 544                     logging.warn("Invalid image file at %s" % last_iconpath)
 545                     continue
 546
 547                 size = dpi_to_px(density)
 548
 549                 im.thumbnail((size, size), Image.ANTIALIAS)
 550                 im.save(iconpath, "PNG")
 551                 empty_densities.remove(density)
 552
 553             # Then just copy from the highest resolution available
 554             last_density = None
 555             for density in reversed(densities):
 556                 if density not in empty_densities:
 557                     last_density = density
 558                     continue
 559                 if last_density is None:
 560                     continue
 561                 logging.info("Density %s not available, copying from lower density %s"
 562                              % (density, last_density))
 563
 564                 shutil.copyfile(
 565                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 566                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 567
 568                 empty_densities.remove(density)
 569
 570             for density in densities:
 571                 icon_dir = get_icon_dir(repodir, density)
 572                 icondest = os.path.join(icon_dir, iconfilename)
 573                 resize_icon(icondest, density)
 574
 575             # Copy from icons-mdpi to icons since mdpi is the baseline density
 576             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 577             if os.path.isfile(baseline):
 578                 shutil.copyfile(baseline,
 579                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 580
 581             # Record in known apks, getting the added date at the same time..
 582             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 583             if added:
 584                 thisinfo['added'] = added
 585
 586             apkcache[apkfilename] = thisinfo
 587             cachechanged = True
 588
 589         apks.append(thisinfo)
 590
 591     return apks, cachechanged
 592
 593
 594 repo_pubkey_fingerprint = None
 595
 596
 597 def make_index(apps, apks, repodir, archive, categories):
 598     """Make a repo index.
 599
 600     :param apps: fully populated apps list
 601     :param apks: full populated apks list
 602     :param repodir: the repo directory
 603     :param archive: True if this is the archive repo, False if it's the
 604                     main one.
 605     :param categories: list of categories
 606     """
 607
 608     doc = Document()
 609
 610     def addElement(name, value, doc, parent):
 611         el = doc.createElement(name)
 612         el.appendChild(doc.createTextNode(value))
 613         parent.appendChild(el)
 614
 615     def addElementCDATA(name, value, doc, parent):
 616         el = doc.createElement(name)
 617         el.appendChild(doc.createCDATASection(value))
 618         parent.appendChild(el)
 619
 620     root = doc.createElement("fdroid")
 621     doc.appendChild(root)
 622
 623     repoel = doc.createElement("repo")
 624
 625     if archive:
 626         repoel.setAttribute("name", config['archive_name'])
 627         if config['repo_maxage'] != 0:
 628             repoel.setAttribute("maxage", str(config['repo_maxage']))
 629         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 630         repoel.setAttribute("url", config['archive_url'])
 631         addElement('description', config['archive_description'], doc, repoel)
 632
 633     else:
 634         repoel.setAttribute("name", config['repo_name'])
 635         if config['repo_maxage'] != 0:
 636             repoel.setAttribute("maxage", str(config['repo_maxage']))
 637         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 638         repoel.setAttribute("url", config['repo_url'])
 639         addElement('description', config['repo_description'], doc, repoel)
 640
 641     repoel.setAttribute("version", "12")
 642     repoel.setAttribute("timestamp", str(int(time.time())))
 643
 644     if 'repo_keyalias' in config:
 645
 646         # Generate a certificate fingerprint the same way keytool does it
 647         # (but with slightly different formatting)
 648         def cert_fingerprint(data):
 649             digest = hashlib.sha256(data).digest()
 650             ret = []
 651             ret.append(' '.join("%02X" % ord(b) for b in digest))
 652             return " ".join(ret)
 653
 654         def extract_pubkey():
 655             p = FDroidPopen(['keytool', '-exportcert',
 656                              '-alias', config['repo_keyalias'],
 657                              '-keystore', config['keystore'],
 658                              '-storepass:file', config['keystorepassfile']]
 659                             + config['smartcardoptions'])
 660             if p.returncode != 0:
 661                 msg = "Failed to get repo pubkey!"
 662                 if config['keystore'] == 'NONE':
 663                     msg += ' Is your crypto smartcard plugged in?'
 664                 logging.critical(msg)
 665                 sys.exit(1)
 666             global repo_pubkey_fingerprint
 667             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 668             return "".join("%02x" % ord(b) for b in p.stdout)
 669
 670         repoel.setAttribute("pubkey", extract_pubkey())
 671
 672     root.appendChild(repoel)
 673
 674     for app in apps:
 675
 676         if app['Disabled'] is not None:
 677             continue
 678
 679         # Get a list of the apks for this app...
 680         apklist = []
 681         for apk in apks:
 682             if apk['id'] == app['id']:
 683                 apklist.append(apk)
 684
 685         if len(apklist) == 0:
 686             continue
 687
 688         apel = doc.createElement("application")
 689         apel.setAttribute("id", app['id'])
 690         root.appendChild(apel)
 691
 692         addElement('id', app['id'], doc, apel)
 693         if 'added' in app:
 694             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 695         if 'lastupdated' in app:
 696             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 697         addElement('name', app['Name'], doc, apel)
 698         addElement('summary', app['Summary'], doc, apel)
 699         if app['icon']:
 700             addElement('icon', app['icon'], doc, apel)
 701
 702         def linkres(link):
 703             for app in apps:
 704                 if app['id'] == link:
 705                     return ("fdroid.app:" + link, app['Name'])
 706             raise MetaDataException("Cannot resolve app id " + link)
 707         addElement('desc',
 708                    metadata.description_html(app['Description'], linkres),
 709                    doc, apel)
 710         addElement('license', app['License'], doc, apel)
 711         if 'Categories' in app:
 712             addElement('categories', ','.join(app["Categories"]), doc, apel)
 713             # We put the first (primary) category in LAST, which will have
 714             # the desired effect of making clients that only understand one
 715             # category see that one.
 716             addElement('category', app["Categories"][0], doc, apel)
 717         addElement('web', app['Web Site'], doc, apel)
 718         addElement('source', app['Source Code'], doc, apel)
 719         addElement('tracker', app['Issue Tracker'], doc, apel)
 720         if app['Donate']:
 721             addElement('donate', app['Donate'], doc, apel)
 722         if app['Bitcoin']:
 723             addElement('bitcoin', app['Bitcoin'], doc, apel)
 724         if app['Litecoin']:
 725             addElement('litecoin', app['Litecoin'], doc, apel)
 726         if app['Dogecoin']:
 727             addElement('dogecoin', app['Dogecoin'], doc, apel)
 728         if app['FlattrID']:
 729             addElement('flattr', app['FlattrID'], doc, apel)
 730
 731         # These elements actually refer to the current version (i.e. which
 732         # one is recommended. They are historically mis-named, and need
 733         # changing, but stay like this for now to support existing clients.
 734         addElement('marketversion', app['Current Version'], doc, apel)
 735         addElement('marketvercode', app['Current Version Code'], doc, apel)
 736
 737         if app['AntiFeatures']:
 738             af = app['AntiFeatures'].split(',')
 739             # TODO: Temporarily not including UpstreamNonFree in the index,
 740             # because current F-Droid clients do not understand it, and also
 741             # look ugly when they encounter an unknown antifeature. This
 742             # filtering can be removed in time...
 743             if 'UpstreamNonFree' in af:
 744                 af.remove('UpstreamNonFree')
 745             if af:
 746                 addElement('antifeatures', ','.join(af), doc, apel)
 747         if app['Provides']:
 748             pv = app['Provides'].split(',')
 749             addElement('provides', ','.join(pv), doc, apel)
 750         if app['Requires Root']:
 751             addElement('requirements', 'root', doc, apel)
 752
 753         # Sort the apk list into version order, just so the web site
 754         # doesn't have to do any work by default...
 755         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 756
 757         # Check for duplicates - they will make the client unhappy...
 758         for i in range(len(apklist) - 1):
 759             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 760                 logging.critical("duplicate versions: '%s' - '%s'" % (
 761                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 762                 sys.exit(1)
 763
 764         for apk in apklist:
 765             apkel = doc.createElement("package")
 766             apel.appendChild(apkel)
 767             addElement('version', apk['version'], doc, apkel)
 768             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 769             addElement('apkname', apk['apkname'], doc, apkel)
 770             if 'srcname' in apk:
 771                 addElement('srcname', apk['srcname'], doc, apkel)
 772             for hash_type in ['sha256']:
 773                 if hash_type not in apk:
 774                     continue
 775                 hashel = doc.createElement("hash")
 776                 hashel.setAttribute("type", hash_type)
 777                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 778                 apkel.appendChild(hashel)
 779             addElement('sig', apk['sig'], doc, apkel)
 780             addElement('size', str(apk['size']), doc, apkel)
 781             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 782             if 'maxsdkversion' in apk:
 783                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 784             if 'added' in apk:
 785                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 786             if app['Requires Root']:
 787                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 788                     apk['permissions'].append('ACCESS_SUPERUSER')
 789
 790             if len(apk['permissions']) > 0:
 791                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 792             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 793                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 794             if len(apk['features']) > 0:
 795                 addElement('features', ','.join(apk['features']), doc, apkel)
 796
 797     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 798     if options.pretty:
 799         output = doc.toprettyxml()
 800     else:
 801         output = doc.toxml()
 802     of.write(output)
 803     of.close()
 804
 805     if 'repo_keyalias' in config:
 806
 807         logging.info("Creating signed index with this key:")
 808         logging.info("SHA256: %s" % repo_pubkey_fingerprint)
 809
 810         # Create a jar of the index...
 811         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 812         if p.returncode != 0:
 813             logging.critical("Failed to create jar file")
 814             sys.exit(1)
 815
 816         # Sign the index...
 817         args = ['jarsigner', '-keystore', config['keystore'],
 818                 '-storepass:file', config['keystorepassfile'],
 819                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 820                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 821         if config['keystore'] == 'NONE':
 822             args += config['smartcardoptions']
 823         else:  # smardcards never use -keypass
 824             args += ['-keypass:file', config['keypassfile']]
 825         p = FDroidPopen(args)
 826         # TODO keypass should be sent via stdin
 827         if p.returncode != 0:
 828             logging.info("Failed to sign index")
 829             sys.exit(1)
 830
 831     # Copy the repo icon into the repo directory...
 832     icon_dir = os.path.join(repodir, 'icons')
 833     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 834     shutil.copyfile(config['repo_icon'], iconfilename)
 835
 836     # Write a category list in the repo to allow quick access...
 837     catdata = ''
 838     for cat in categories:
 839         catdata += cat + '\n'
 840     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 841     f.write(catdata)
 842     f.close()
 843
 844
 845 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 846
 847     for app in apps:
 848
 849         # Get a list of the apks for this app...
 850         apklist = []
 851         for apk in apks:
 852             if apk['id'] == app['id']:
 853                 apklist.append(apk)
 854
 855         # Sort the apk list into version order...
 856         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 857
 858         if app['Archive Policy']:
 859             keepversions = int(app['Archive Policy'][:-9])
 860         else:
 861             keepversions = defaultkeepversions
 862
 863         if len(apklist) > keepversions:
 864             for apk in apklist[keepversions:]:
 865                 logging.info("Moving " + apk['apkname'] + " to archive")
 866                 shutil.move(os.path.join(repodir, apk['apkname']),
 867                             os.path.join(archivedir, apk['apkname']))
 868                 if 'srcname' in apk:
 869                     shutil.move(os.path.join(repodir, apk['srcname']),
 870                                 os.path.join(archivedir, apk['srcname']))
 871                 archapks.append(apk)
 872                 apks.remove(apk)
 873
 874
 875 config = None
 876 options = None
 877
 878
 879 def main():
 880
 881     global config, options
 882
 883     # Parse command line...
 884     parser = OptionParser()
 885     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 886                       help="Create skeleton metadata files that are missing")
 887     parser.add_option("--delete-unknown", action="store_true", default=False,
 888                       help="Delete APKs without metadata from the repo")
 889     parser.add_option("-v", "--verbose", action="store_true", default=False,
 890                       help="Spew out even more information than normal")
 891     parser.add_option("-q", "--quiet", action="store_true", default=False,
 892                       help="Restrict output to warnings and errors")
 893     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 894                       help="Report on build data status")
 895     parser.add_option("-i", "--interactive", default=False, action="store_true",
 896                       help="Interactively ask about things that need updating.")
 897     parser.add_option("-I", "--icons", action="store_true", default=False,
 898                       help="Resize all the icons exceeding the max pixel size and exit")
 899     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 900                       help="Specify editor to use in interactive mode. Default " +
 901                       "is /etc/alternatives/editor")
 902     parser.add_option("-w", "--wiki", default=False, action="store_true",
 903                       help="Update the wiki")
 904     parser.add_option("", "--pretty", action="store_true", default=False,
 905                       help="Produce human-readable index.xml")
 906     parser.add_option("--clean", action="store_true", default=False,
 907                       help="Clean update - don't uses caches, reprocess all apks")
 908     (options, args) = parser.parse_args()
 909
 910     config = common.read_config(options)
 911
 912     repodirs = ['repo']
 913     if config['archive_older'] != 0:
 914         repodirs.append('archive')
 915         if not os.path.exists('archive'):
 916             os.mkdir('archive')
 917
 918     if options.icons:
 919         resize_all_icons(repodirs)
 920         sys.exit(0)
 921
 922     # Get all apps...
 923     apps = metadata.read_metadata()
 924
 925     # Generate a list of categories...
 926     categories = set()
 927     for app in apps:
 928         categories.update(app['Categories'])
 929
 930     # Read known apks data (will be updated and written back when we've finished)
 931     knownapks = common.KnownApks()
 932
 933     # Gather information about all the apk files in the repo directory, using
 934     # cached data if possible.
 935     apkcachefile = os.path.join('tmp', 'apkcache')
 936     if not options.clean and os.path.exists(apkcachefile):
 937         with open(apkcachefile, 'rb') as cf:
 938             apkcache = pickle.load(cf)
 939     else:
 940         apkcache = {}
 941     cachechanged = False
 942
 943     delete_disabled_builds(apps, apkcache, repodirs)
 944
 945     # Scan all apks in the main repo
 946     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 947     if cc:
 948         cachechanged = True
 949
 950     # Generate warnings for apk's with no metadata (or create skeleton
 951     # metadata files, if requested on the command line)
 952     newmetadata = False
 953     for apk in apks:
 954         found = False
 955         for app in apps:
 956             if app['id'] == apk['id']:
 957                 found = True
 958                 break
 959         if not found:
 960             if options.create_metadata:
 961                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 962                 f.write("License:Unknown\n")
 963                 f.write("Web Site:\n")
 964                 f.write("Source Code:\n")
 965                 f.write("Issue Tracker:\n")
 966                 f.write("Summary:" + apk['name'] + "\n")
 967                 f.write("Description:\n")
 968                 f.write(apk['name'] + "\n")
 969                 f.write(".\n")
 970                 f.close()
 971                 logging.info("Generated skeleton metadata for " + apk['id'])
 972                 newmetadata = True
 973             else:
 974                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
 975                 if options.delete_unknown:
 976                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
 977                     rmf = os.path.join(repodirs[0], apk['apkname'])
 978                     if not os.path.exists(rmf):
 979                         logging.error("Could not find {0} to remove it".format(rmf))
 980                     else:
 981                         os.remove(rmf)
 982                 else:
 983                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
 984
 985     # update the metadata with the newly created ones included
 986     if newmetadata:
 987         apps = metadata.read_metadata()
 988
 989     # Scan the archive repo for apks as well
 990     if len(repodirs) > 1:
 991         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 992         if cc:
 993             cachechanged = True
 994     else:
 995         archapks = []
 996
 997     # Some information from the apks needs to be applied up to the application
 998     # level. When doing this, we use the info from the most recent version's apk.
 999     # We deal with figuring out when the app was added and last updated at the
1000     # same time.
1001     for app in apps:
1002         bestver = 0
1003         added = None
1004         lastupdated = None
1005         for apk in apks + archapks:
1006             if apk['id'] == app['id']:
1007                 if apk['versioncode'] > bestver:
1008                     bestver = apk['versioncode']
1009                     bestapk = apk
1010
1011                 if 'added' in apk:
1012                     if not added or apk['added'] < added:
1013                         added = apk['added']
1014                     if not lastupdated or apk['added'] > lastupdated:
1015                         lastupdated = apk['added']
1016
1017         if added:
1018             app['added'] = added
1019         else:
1020             logging.warn("Don't know when " + app['id'] + " was added")
1021         if lastupdated:
1022             app['lastupdated'] = lastupdated
1023         else:
1024             logging.warn("Don't know when " + app['id'] + " was last updated")
1025
1026         if bestver == 0:
1027             if app['Name'] is None:
1028                 app['Name'] = app['id']
1029             app['icon'] = None
1030             logging.warn("Application " + app['id'] + " has no packages")
1031         else:
1032             if app['Name'] is None:
1033                 app['Name'] = bestapk['name']
1034             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1035
1036     # Sort the app list by name, then the web site doesn't have to by default.
1037     # (we had to wait until we'd scanned the apks to do this, because mostly the
1038     # name comes from there!)
1039     apps = sorted(apps, key=lambda app: app['Name'].upper())
1040
1041     if len(repodirs) > 1:
1042         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1043
1044     # Make the index for the main repo...
1045     make_index(apps, apks, repodirs[0], False, categories)
1046
1047     # If there's an archive repo,  make the index for it. We already scanned it
1048     # earlier on.
1049     if len(repodirs) > 1:
1050         make_index(apps, archapks, repodirs[1], True, categories)
1051
1052     if config['update_stats']:
1053
1054         # Update known apks info...
1055         knownapks.writeifchanged()
1056
1057         # Generate latest apps data for widget
1058         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1059             data = ''
1060             for line in file(os.path.join('stats', 'latestapps.txt')):
1061                 appid = line.rstrip()
1062                 data += appid + "\t"
1063                 for app in apps:
1064                     if app['id'] == appid:
1065                         data += app['Name'] + "\t"
1066                         if app['icon'] is not None:
1067                             data += app['icon'] + "\t"
1068                         data += app['License'] + "\n"
1069                         break
1070             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1071             f.write(data)
1072             f.close()
1073
1074     if cachechanged:
1075         with open(apkcachefile, 'wb') as cf:
1076             pickle.dump(apkcache, cf)
1077
1078     # Update the wiki...
1079     if options.wiki:
1080         update_wiki(apps, apks + archapks)
1081
1082     logging.info("Finished.")
1083
1084 if __name__ == "__main__":
1085     main()