fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304     except Exception, e:
 305         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 306
 307
 308 def resize_all_icons(repodirs):
 309     """Resize all icons that exceed the max size
 310
 311     :param repodirs: the repo directories to process
 312     """
 313     for repodir in repodirs:
 314         for density in get_densities():
 315             icon_dir = get_icon_dir(repodir, density)
 316             icon_glob = os.path.join(icon_dir, '*.png')
 317             for iconpath in glob.glob(icon_glob):
 318                 resize_icon(iconpath, density)
 319
 320
 321 def scan_apks(apps, apkcache, repodir, knownapks):
 322     """Scan the apks in the given repo directory.
 323
 324     This also extracts the icons.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodir: repo directory to scan
 329     :param knownapks: known apks info
 330     :returns: (apks, cachechanged) where apks is a list of apk information,
 331               and cachechanged is True if the apkcache got changed.
 332     """
 333
 334     cachechanged = False
 335
 336     icon_dirs = get_icon_dirs(repodir)
 337     for icon_dir in icon_dirs:
 338         if os.path.exists(icon_dir):
 339             if options.clean:
 340                 shutil.rmtree(icon_dir)
 341                 os.makedirs(icon_dir)
 342         else:
 343             os.makedirs(icon_dir)
 344
 345     apks = []
 346     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 347     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 348     vername_pat = re.compile(".*versionName='([^']*)'.*")
 349     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 350     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 351     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 352     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 353     string_pat = re.compile(".*'([^']*)'.*")
 354     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 355
 356         apkfilename = apkfile[len(repodir) + 1:]
 357         if ' ' in apkfilename:
 358             logging.error("No spaces in APK filenames!")
 359             sys.exit(1)
 360
 361         if apkfilename in apkcache:
 362             logging.debug("Reading " + apkfilename + " from cache")
 363             thisinfo = apkcache[apkfilename]
 364
 365         else:
 366             if options.verbose:
 367                 logging.info("Processing " + apkfilename)
 368             thisinfo = {}
 369             thisinfo['apkname'] = apkfilename
 370             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 371             if os.path.exists(os.path.join(repodir, srcfilename)):
 372                 thisinfo['srcname'] = srcfilename
 373             thisinfo['size'] = os.path.getsize(apkfile)
 374             thisinfo['permissions'] = []
 375             thisinfo['features'] = []
 376             thisinfo['icons_src'] = {}
 377             thisinfo['icons'] = {}
 378             p = FDroidPopen([os.path.join(config['sdk_path'], 'build-tools',
 379                                           config['build_tools'], 'aapt'),
 380                              'dump', 'badging', apkfile])
 381             if p.returncode != 0:
 382                 if options.delete_unknown:
 383                     if os.path.exists(apkfile):
 384                         logging.error("Failed to get apk information, deleting " + apkfile)
 385                         os.remove(apkfile)
 386                     else:
 387                         logging.error("Could not find {0} to remove it".format(apkfile))
 388                 else:
 389                     logging.error("Failed to get apk information, skipping " + apkfile)
 390                 continue
 391             for line in p.stdout.splitlines():
 392                 if line.startswith("package:"):
 393                     try:
 394                         thisinfo['id'] = re.match(name_pat, line).group(1)
 395                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 396                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 397                     except Exception, e:
 398                         logging.info("Package matching failed: " + str(e))
 399                         logging.info("Line was: " + line)
 400                         sys.exit(1)
 401                 elif line.startswith("application:"):
 402                     thisinfo['name'] = re.match(label_pat, line).group(1)
 403                     # Keep path to non-dpi icon in case we need it
 404                     match = re.match(icon_pat_nodpi, line)
 405                     if match:
 406                         thisinfo['icons_src']['-1'] = match.group(1)
 407                 elif line.startswith("launchable-activity:"):
 408                     # Only use launchable-activity as fallback to application
 409                     if not thisinfo['name']:
 410                         thisinfo['name'] = re.match(label_pat, line).group(1)
 411                     if '-1' not in thisinfo['icons_src']:
 412                         match = re.match(icon_pat_nodpi, line)
 413                         if match:
 414                             thisinfo['icons_src']['-1'] = match.group(1)
 415                 elif line.startswith("application-icon-"):
 416                     match = re.match(icon_pat, line)
 417                     if match:
 418                         density = match.group(1)
 419                         path = match.group(2)
 420                         thisinfo['icons_src'][density] = path
 421                 elif line.startswith("sdkVersion:"):
 422                     thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1)
 423                 elif line.startswith("maxSdkVersion:"):
 424                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 425                 elif line.startswith("native-code:"):
 426                     thisinfo['nativecode'] = []
 427                     for arch in line[13:].split(' '):
 428                         thisinfo['nativecode'].append(arch[1:-1])
 429                 elif line.startswith("uses-permission:"):
 430                     perm = re.match(string_pat, line).group(1)
 431                     if perm.startswith("android.permission."):
 432                         perm = perm[19:]
 433                     thisinfo['permissions'].append(perm)
 434                 elif line.startswith("uses-feature:"):
 435                     perm = re.match(string_pat, line).group(1)
 436                     # Filter out this, it's only added with the latest SDK tools and
 437                     # causes problems for lots of apps.
 438                     if perm != "android.hardware.screen.portrait" \
 439                             and perm != "android.hardware.screen.landscape":
 440                         if perm.startswith("android.feature."):
 441                             perm = perm[16:]
 442                         thisinfo['features'].append(perm)
 443
 444             if 'sdkversion' not in thisinfo:
 445                 logging.warn("no SDK version information found")
 446                 thisinfo['sdkversion'] = 0
 447
 448             # Check for debuggable apks...
 449             if common.isApkDebuggable(apkfile, config):
 450                 logging.warn("{0} is debuggable... {1}".format(apkfile, line))
 451
 452             # Calculate the sha256...
 453             sha = hashlib.sha256()
 454             with open(apkfile, 'rb') as f:
 455                 while True:
 456                     t = f.read(1024)
 457                     if len(t) == 0:
 458                         break
 459                     sha.update(t)
 460                 thisinfo['sha256'] = sha.hexdigest()
 461
 462             # Get the signature (or md5 of, to be precise)...
 463             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 464             if not os.path.exists(getsig_dir + "/getsig.class"):
 465                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 466                 sys.exit(1)
 467             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 468                              'getsig', os.path.join(os.getcwd(), apkfile)])
 469             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 470                 logging.critical("Failed to get apk signature")
 471                 sys.exit(1)
 472             thisinfo['sig'] = p.stdout[7:].strip()
 473
 474             apk = zipfile.ZipFile(apkfile, 'r')
 475
 476             iconfilename = "%s.%s.png" % (
 477                 thisinfo['id'],
 478                 thisinfo['versioncode'])
 479
 480             # Extract the icon file...
 481             densities = get_densities()
 482             empty_densities = []
 483             for density in densities:
 484                 if density not in thisinfo['icons_src']:
 485                     empty_densities.append(density)
 486                     continue
 487                 iconsrc = thisinfo['icons_src'][density]
 488                 icon_dir = get_icon_dir(repodir, density)
 489                 icondest = os.path.join(icon_dir, iconfilename)
 490
 491                 try:
 492                     iconfile = open(icondest, 'wb')
 493                     iconfile.write(apk.read(iconsrc))
 494                     iconfile.close()
 495                     thisinfo['icons'][density] = iconfilename
 496
 497                 except:
 498                     logging.warn("Error retrieving icon file")
 499                     del thisinfo['icons'][density]
 500                     del thisinfo['icons_src'][density]
 501                     empty_densities.append(density)
 502
 503             if '-1' in thisinfo['icons_src']:
 504                 iconsrc = thisinfo['icons_src']['-1']
 505                 iconpath = os.path.join(
 506                     get_icon_dir(repodir, None), iconfilename)
 507                 iconfile = open(iconpath, 'wb')
 508                 iconfile.write(apk.read(iconsrc))
 509                 iconfile.close()
 510                 try:
 511                     im = Image.open(iconpath)
 512                     dpi = px_to_dpi(im.size[0])
 513                     for density in densities:
 514                         if density in thisinfo['icons']:
 515                             break
 516                         if density == densities[-1] or dpi >= int(density):
 517                             thisinfo['icons'][density] = iconfilename
 518                             shutil.move(iconpath,
 519                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 520                             empty_densities.remove(density)
 521                             break
 522                 except Exception, e:
 523                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 524
 525             if thisinfo['icons']:
 526                 thisinfo['icon'] = iconfilename
 527
 528             apk.close()
 529
 530             # First try resizing down to not lose quality
 531             last_density = None
 532             for density in densities:
 533                 if density not in empty_densities:
 534                     last_density = density
 535                     continue
 536                 if last_density is None:
 537                     continue
 538                 logging.info("Density %s not available, resizing down from %s"
 539                              % (density, last_density))
 540
 541                 last_iconpath = os.path.join(
 542                     get_icon_dir(repodir, last_density), iconfilename)
 543                 iconpath = os.path.join(
 544                     get_icon_dir(repodir, density), iconfilename)
 545                 try:
 546                     im = Image.open(last_iconpath)
 547                 except:
 548                     logging.warn("Invalid image file at %s" % last_iconpath)
 549                     continue
 550
 551                 size = dpi_to_px(density)
 552
 553                 im.thumbnail((size, size), Image.ANTIALIAS)
 554                 im.save(iconpath, "PNG")
 555                 empty_densities.remove(density)
 556
 557             # Then just copy from the highest resolution available
 558             last_density = None
 559             for density in reversed(densities):
 560                 if density not in empty_densities:
 561                     last_density = density
 562                     continue
 563                 if last_density is None:
 564                     continue
 565                 logging.info("Density %s not available, copying from lower density %s"
 566                              % (density, last_density))
 567
 568                 shutil.copyfile(
 569                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 570                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 571
 572                 empty_densities.remove(density)
 573
 574             for density in densities:
 575                 icon_dir = get_icon_dir(repodir, density)
 576                 icondest = os.path.join(icon_dir, iconfilename)
 577                 resize_icon(icondest, density)
 578
 579             # Copy from icons-mdpi to icons since mdpi is the baseline density
 580             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 581             if os.path.isfile(baseline):
 582                 shutil.copyfile(baseline,
 583                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 584
 585             # Record in known apks, getting the added date at the same time..
 586             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 587             if added:
 588                 thisinfo['added'] = added
 589
 590             apkcache[apkfilename] = thisinfo
 591             cachechanged = True
 592
 593         apks.append(thisinfo)
 594
 595     return apks, cachechanged
 596
 597
 598 repo_pubkey_fingerprint = None
 599
 600
 601 def make_index(apps, apks, repodir, archive, categories):
 602     """Make a repo index.
 603
 604     :param apps: fully populated apps list
 605     :param apks: full populated apks list
 606     :param repodir: the repo directory
 607     :param archive: True if this is the archive repo, False if it's the
 608                     main one.
 609     :param categories: list of categories
 610     """
 611
 612     doc = Document()
 613
 614     def addElement(name, value, doc, parent):
 615         el = doc.createElement(name)
 616         el.appendChild(doc.createTextNode(value))
 617         parent.appendChild(el)
 618
 619     def addElementCDATA(name, value, doc, parent):
 620         el = doc.createElement(name)
 621         el.appendChild(doc.createCDATASection(value))
 622         parent.appendChild(el)
 623
 624     root = doc.createElement("fdroid")
 625     doc.appendChild(root)
 626
 627     repoel = doc.createElement("repo")
 628
 629     if archive:
 630         repoel.setAttribute("name", config['archive_name'])
 631         if config['repo_maxage'] != 0:
 632             repoel.setAttribute("maxage", str(config['repo_maxage']))
 633         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 634         repoel.setAttribute("url", config['archive_url'])
 635         addElement('description', config['archive_description'], doc, repoel)
 636
 637     else:
 638         repoel.setAttribute("name", config['repo_name'])
 639         if config['repo_maxage'] != 0:
 640             repoel.setAttribute("maxage", str(config['repo_maxage']))
 641         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 642         repoel.setAttribute("url", config['repo_url'])
 643         addElement('description', config['repo_description'], doc, repoel)
 644
 645     repoel.setAttribute("version", "12")
 646     repoel.setAttribute("timestamp", str(int(time.time())))
 647
 648     if 'repo_keyalias' in config:
 649
 650         # Generate a certificate fingerprint the same way keytool does it
 651         # (but with slightly different formatting)
 652         def cert_fingerprint(data):
 653             digest = hashlib.sha256(data).digest()
 654             ret = []
 655             ret.append(' '.join("%02X" % ord(b) for b in digest))
 656             return " ".join(ret)
 657
 658         def extract_pubkey():
 659             p = FDroidPopen(['keytool', '-exportcert',
 660                              '-alias', config['repo_keyalias'],
 661                              '-keystore', config['keystore'],
 662                              '-storepass:file', config['keystorepassfile']]
 663                             + config['smartcardoptions'])
 664             if p.returncode != 0:
 665                 msg = "Failed to get repo pubkey!"
 666                 if config['keystore'] == 'NONE':
 667                     msg += ' Is your crypto smartcard plugged in?'
 668                 logging.critical(msg)
 669                 sys.exit(1)
 670             global repo_pubkey_fingerprint
 671             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 672             return "".join("%02x" % ord(b) for b in p.stdout)
 673
 674         repoel.setAttribute("pubkey", extract_pubkey())
 675
 676     root.appendChild(repoel)
 677
 678     for app in apps:
 679
 680         if app['Disabled'] is not None:
 681             continue
 682
 683         # Get a list of the apks for this app...
 684         apklist = []
 685         for apk in apks:
 686             if apk['id'] == app['id']:
 687                 apklist.append(apk)
 688
 689         if len(apklist) == 0:
 690             continue
 691
 692         apel = doc.createElement("application")
 693         apel.setAttribute("id", app['id'])
 694         root.appendChild(apel)
 695
 696         addElement('id', app['id'], doc, apel)
 697         if 'added' in app:
 698             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 699         if 'lastupdated' in app:
 700             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 701         addElement('name', app['Name'], doc, apel)
 702         addElement('summary', app['Summary'], doc, apel)
 703         if app['icon']:
 704             addElement('icon', app['icon'], doc, apel)
 705
 706         def linkres(link):
 707             for app in apps:
 708                 if app['id'] == link:
 709                     return ("fdroid.app:" + link, app['Name'])
 710             raise MetaDataException("Cannot resolve app id " + link)
 711         addElement('desc',
 712                    metadata.description_html(app['Description'], linkres),
 713                    doc, apel)
 714         addElement('license', app['License'], doc, apel)
 715         if 'Categories' in app:
 716             addElement('categories', ','.join(app["Categories"]), doc, apel)
 717             # We put the first (primary) category in LAST, which will have
 718             # the desired effect of making clients that only understand one
 719             # category see that one.
 720             addElement('category', app["Categories"][0], doc, apel)
 721         addElement('web', app['Web Site'], doc, apel)
 722         addElement('source', app['Source Code'], doc, apel)
 723         addElement('tracker', app['Issue Tracker'], doc, apel)
 724         if app['Donate']:
 725             addElement('donate', app['Donate'], doc, apel)
 726         if app['Bitcoin']:
 727             addElement('bitcoin', app['Bitcoin'], doc, apel)
 728         if app['Litecoin']:
 729             addElement('litecoin', app['Litecoin'], doc, apel)
 730         if app['Dogecoin']:
 731             addElement('dogecoin', app['Dogecoin'], doc, apel)
 732         if app['FlattrID']:
 733             addElement('flattr', app['FlattrID'], doc, apel)
 734
 735         # These elements actually refer to the current version (i.e. which
 736         # one is recommended. They are historically mis-named, and need
 737         # changing, but stay like this for now to support existing clients.
 738         addElement('marketversion', app['Current Version'], doc, apel)
 739         addElement('marketvercode', app['Current Version Code'], doc, apel)
 740
 741         if app['AntiFeatures']:
 742             af = app['AntiFeatures'].split(',')
 743             # TODO: Temporarily not including UpstreamNonFree in the index,
 744             # because current F-Droid clients do not understand it, and also
 745             # look ugly when they encounter an unknown antifeature. This
 746             # filtering can be removed in time...
 747             if 'UpstreamNonFree' in af:
 748                 af.remove('UpstreamNonFree')
 749             if af:
 750                 addElement('antifeatures', ','.join(af), doc, apel)
 751         if app['Provides']:
 752             pv = app['Provides'].split(',')
 753             addElement('provides', ','.join(pv), doc, apel)
 754         if app['Requires Root']:
 755             addElement('requirements', 'root', doc, apel)
 756
 757         # Sort the apk list into version order, just so the web site
 758         # doesn't have to do any work by default...
 759         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 760
 761         # Check for duplicates - they will make the client unhappy...
 762         for i in range(len(apklist) - 1):
 763             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 764                 logging.critical("duplicate versions: '%s' - '%s'" % (
 765                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 766                 sys.exit(1)
 767
 768         for apk in apklist:
 769             apkel = doc.createElement("package")
 770             apel.appendChild(apkel)
 771             addElement('version', apk['version'], doc, apkel)
 772             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 773             addElement('apkname', apk['apkname'], doc, apkel)
 774             if 'srcname' in apk:
 775                 addElement('srcname', apk['srcname'], doc, apkel)
 776             for hash_type in ['sha256']:
 777                 if hash_type not in apk:
 778                     continue
 779                 hashel = doc.createElement("hash")
 780                 hashel.setAttribute("type", hash_type)
 781                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 782                 apkel.appendChild(hashel)
 783             addElement('sig', apk['sig'], doc, apkel)
 784             addElement('size', str(apk['size']), doc, apkel)
 785             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 786             if 'maxsdkversion' in apk:
 787                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 788             if 'added' in apk:
 789                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 790             if app['Requires Root']:
 791                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 792                     apk['permissions'].append('ACCESS_SUPERUSER')
 793
 794             if len(apk['permissions']) > 0:
 795                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 796             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 797                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 798             if len(apk['features']) > 0:
 799                 addElement('features', ','.join(apk['features']), doc, apkel)
 800
 801     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 802     if options.pretty:
 803         output = doc.toprettyxml()
 804     else:
 805         output = doc.toxml()
 806     of.write(output)
 807     of.close()
 808
 809     if 'repo_keyalias' in config:
 810
 811         logging.info("Creating signed index with this key (SHA256):")
 812         logging.info("%s" % repo_pubkey_fingerprint)
 813
 814         # Create a jar of the index...
 815         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 816         if p.returncode != 0:
 817             logging.critical("Failed to create jar file")
 818             sys.exit(1)
 819
 820         # Sign the index...
 821         args = ['jarsigner', '-keystore', config['keystore'],
 822                 '-storepass:file', config['keystorepassfile'],
 823                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 824                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 825         if config['keystore'] == 'NONE':
 826             args += config['smartcardoptions']
 827         else:  # smardcards never use -keypass
 828             args += ['-keypass:file', config['keypassfile']]
 829         p = FDroidPopen(args)
 830         # TODO keypass should be sent via stdin
 831         if p.returncode != 0:
 832             logging.info("Failed to sign index")
 833             sys.exit(1)
 834
 835     # Copy the repo icon into the repo directory...
 836     icon_dir = os.path.join(repodir, 'icons')
 837     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 838     shutil.copyfile(config['repo_icon'], iconfilename)
 839
 840     # Write a category list in the repo to allow quick access...
 841     catdata = ''
 842     for cat in categories:
 843         catdata += cat + '\n'
 844     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 845     f.write(catdata)
 846     f.close()
 847
 848
 849 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 850
 851     for app in apps:
 852
 853         # Get a list of the apks for this app...
 854         apklist = []
 855         for apk in apks:
 856             if apk['id'] == app['id']:
 857                 apklist.append(apk)
 858
 859         # Sort the apk list into version order...
 860         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 861
 862         if app['Archive Policy']:
 863             keepversions = int(app['Archive Policy'][:-9])
 864         else:
 865             keepversions = defaultkeepversions
 866
 867         if len(apklist) > keepversions:
 868             for apk in apklist[keepversions:]:
 869                 logging.info("Moving " + apk['apkname'] + " to archive")
 870                 shutil.move(os.path.join(repodir, apk['apkname']),
 871                             os.path.join(archivedir, apk['apkname']))
 872                 if 'srcname' in apk:
 873                     shutil.move(os.path.join(repodir, apk['srcname']),
 874                                 os.path.join(archivedir, apk['srcname']))
 875                 archapks.append(apk)
 876                 apks.remove(apk)
 877
 878
 879 config = None
 880 options = None
 881
 882
 883 def main():
 884
 885     global config, options
 886
 887     # Parse command line...
 888     parser = OptionParser()
 889     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 890                       help="Create skeleton metadata files that are missing")
 891     parser.add_option("--delete-unknown", action="store_true", default=False,
 892                       help="Delete APKs without metadata from the repo")
 893     parser.add_option("-v", "--verbose", action="store_true", default=False,
 894                       help="Spew out even more information than normal")
 895     parser.add_option("-q", "--quiet", action="store_true", default=False,
 896                       help="Restrict output to warnings and errors")
 897     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 898                       help="Report on build data status")
 899     parser.add_option("-i", "--interactive", default=False, action="store_true",
 900                       help="Interactively ask about things that need updating.")
 901     parser.add_option("-I", "--icons", action="store_true", default=False,
 902                       help="Resize all the icons exceeding the max pixel size and exit")
 903     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 904                       help="Specify editor to use in interactive mode. Default " +
 905                       "is /etc/alternatives/editor")
 906     parser.add_option("-w", "--wiki", default=False, action="store_true",
 907                       help="Update the wiki")
 908     parser.add_option("", "--pretty", action="store_true", default=False,
 909                       help="Produce human-readable index.xml")
 910     parser.add_option("--clean", action="store_true", default=False,
 911                       help="Clean update - don't uses caches, reprocess all apks")
 912     (options, args) = parser.parse_args()
 913
 914     config = common.read_config(options)
 915
 916     repodirs = ['repo']
 917     if config['archive_older'] != 0:
 918         repodirs.append('archive')
 919         if not os.path.exists('archive'):
 920             os.mkdir('archive')
 921
 922     if options.icons:
 923         resize_all_icons(repodirs)
 924         sys.exit(0)
 925
 926     # Get all apps...
 927     apps = metadata.read_metadata()
 928
 929     # Generate a list of categories...
 930     categories = set()
 931     for app in apps:
 932         categories.update(app['Categories'])
 933
 934     # Read known apks data (will be updated and written back when we've finished)
 935     knownapks = common.KnownApks()
 936
 937     # Gather information about all the apk files in the repo directory, using
 938     # cached data if possible.
 939     apkcachefile = os.path.join('tmp', 'apkcache')
 940     if not options.clean and os.path.exists(apkcachefile):
 941         with open(apkcachefile, 'rb') as cf:
 942             apkcache = pickle.load(cf)
 943     else:
 944         apkcache = {}
 945     cachechanged = False
 946
 947     delete_disabled_builds(apps, apkcache, repodirs)
 948
 949     # Scan all apks in the main repo
 950     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 951     if cc:
 952         cachechanged = True
 953
 954     # Generate warnings for apk's with no metadata (or create skeleton
 955     # metadata files, if requested on the command line)
 956     newmetadata = False
 957     for apk in apks:
 958         found = False
 959         for app in apps:
 960             if app['id'] == apk['id']:
 961                 found = True
 962                 break
 963         if not found:
 964             if options.create_metadata:
 965                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 966                 f.write("License:Unknown\n")
 967                 f.write("Web Site:\n")
 968                 f.write("Source Code:\n")
 969                 f.write("Issue Tracker:\n")
 970                 f.write("Summary:" + apk['name'] + "\n")
 971                 f.write("Description:\n")
 972                 f.write(apk['name'] + "\n")
 973                 f.write(".\n")
 974                 f.close()
 975                 logging.info("Generated skeleton metadata for " + apk['id'])
 976                 newmetadata = True
 977             else:
 978                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
 979                 if options.delete_unknown:
 980                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
 981                     rmf = os.path.join(repodirs[0], apk['apkname'])
 982                     if not os.path.exists(rmf):
 983                         logging.error("Could not find {0} to remove it".format(rmf))
 984                     else:
 985                         os.remove(rmf)
 986                 else:
 987                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
 988
 989     # update the metadata with the newly created ones included
 990     if newmetadata:
 991         apps = metadata.read_metadata()
 992
 993     # Scan the archive repo for apks as well
 994     if len(repodirs) > 1:
 995         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
 996         if cc:
 997             cachechanged = True
 998     else:
 999         archapks = []
1000
1001     # Some information from the apks needs to be applied up to the application
1002     # level. When doing this, we use the info from the most recent version's apk.
1003     # We deal with figuring out when the app was added and last updated at the
1004     # same time.
1005     for app in apps:
1006         bestver = 0
1007         added = None
1008         lastupdated = None
1009         for apk in apks + archapks:
1010             if apk['id'] == app['id']:
1011                 if apk['versioncode'] > bestver:
1012                     bestver = apk['versioncode']
1013                     bestapk = apk
1014
1015                 if 'added' in apk:
1016                     if not added or apk['added'] < added:
1017                         added = apk['added']
1018                     if not lastupdated or apk['added'] > lastupdated:
1019                         lastupdated = apk['added']
1020
1021         if added:
1022             app['added'] = added
1023         else:
1024             logging.warn("Don't know when " + app['id'] + " was added")
1025         if lastupdated:
1026             app['lastupdated'] = lastupdated
1027         else:
1028             logging.warn("Don't know when " + app['id'] + " was last updated")
1029
1030         if bestver == 0:
1031             if app['Name'] is None:
1032                 app['Name'] = app['id']
1033             app['icon'] = None
1034             logging.warn("Application " + app['id'] + " has no packages")
1035         else:
1036             if app['Name'] is None:
1037                 app['Name'] = bestapk['name']
1038             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1039
1040     # Sort the app list by name, then the web site doesn't have to by default.
1041     # (we had to wait until we'd scanned the apks to do this, because mostly the
1042     # name comes from there!)
1043     apps = sorted(apps, key=lambda app: app['Name'].upper())
1044
1045     if len(repodirs) > 1:
1046         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1047
1048     # Make the index for the main repo...
1049     make_index(apps, apks, repodirs[0], False, categories)
1050
1051     # If there's an archive repo,  make the index for it. We already scanned it
1052     # earlier on.
1053     if len(repodirs) > 1:
1054         make_index(apps, archapks, repodirs[1], True, categories)
1055
1056     if config['update_stats']:
1057
1058         # Update known apks info...
1059         knownapks.writeifchanged()
1060
1061         # Generate latest apps data for widget
1062         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1063             data = ''
1064             for line in file(os.path.join('stats', 'latestapps.txt')):
1065                 appid = line.rstrip()
1066                 data += appid + "\t"
1067                 for app in apps:
1068                     if app['id'] == appid:
1069                         data += app['Name'] + "\t"
1070                         if app['icon'] is not None:
1071                             data += app['icon'] + "\t"
1072                         data += app['License'] + "\n"
1073                         break
1074             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1075             f.write(data)
1076             f.close()
1077
1078     if cachechanged:
1079         with open(apkcachefile, 'wb') as cf:
1080             pickle.dump(apkcache, cf)
1081
1082     # Update the wiki...
1083     if options.wiki:
1084         update_wiki(apps, apks + archapks)
1085
1086     logging.info("Finished.")
1087
1088 if __name__ == "__main__":
1089     main()