fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen, SilentPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(app['id'], thisbuild['vercode'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304     except Exception, e:
 305         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 306
 307
 308 def resize_all_icons(repodirs):
 309     """Resize all icons that exceed the max size
 310
 311     :param repodirs: the repo directories to process
 312     """
 313     for repodir in repodirs:
 314         for density in get_densities():
 315             icon_dir = get_icon_dir(repodir, density)
 316             icon_glob = os.path.join(icon_dir, '*.png')
 317             for iconpath in glob.glob(icon_glob):
 318                 resize_icon(iconpath, density)
 319
 320
 321 def scan_apks(apps, apkcache, repodir, knownapks):
 322     """Scan the apks in the given repo directory.
 323
 324     This also extracts the icons.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodir: repo directory to scan
 329     :param knownapks: known apks info
 330     :returns: (apks, cachechanged) where apks is a list of apk information,
 331               and cachechanged is True if the apkcache got changed.
 332     """
 333
 334     cachechanged = False
 335
 336     icon_dirs = get_icon_dirs(repodir)
 337     for icon_dir in icon_dirs:
 338         if os.path.exists(icon_dir):
 339             if options.clean:
 340                 shutil.rmtree(icon_dir)
 341                 os.makedirs(icon_dir)
 342         else:
 343             os.makedirs(icon_dir)
 344
 345     apks = []
 346     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 347     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 348     vername_pat = re.compile(".*versionName='([^']*)'.*")
 349     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 350     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 351     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 352     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 353     string_pat = re.compile(".*'([^']*)'.*")
 354     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 355
 356         apkfilename = apkfile[len(repodir) + 1:]
 357         if ' ' in apkfilename:
 358             logging.critical("Spaces in filenames are not allowed.")
 359             sys.exit(1)
 360
 361         if apkfilename in apkcache:
 362             logging.debug("Reading " + apkfilename + " from cache")
 363             thisinfo = apkcache[apkfilename]
 364
 365         else:
 366             logging.debug("Processing " + apkfilename)
 367             thisinfo = {}
 368             thisinfo['apkname'] = apkfilename
 369             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 370             if os.path.exists(os.path.join(repodir, srcfilename)):
 371                 thisinfo['srcname'] = srcfilename
 372             thisinfo['size'] = os.path.getsize(apkfile)
 373             thisinfo['permissions'] = set()
 374             thisinfo['features'] = set()
 375             thisinfo['icons_src'] = {}
 376             thisinfo['icons'] = {}
 377             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 378             if p.returncode != 0:
 379                 if options.delete_unknown:
 380                     if os.path.exists(apkfile):
 381                         logging.error("Failed to get apk information, deleting " + apkfile)
 382                         os.remove(apkfile)
 383                     else:
 384                         logging.error("Could not find {0} to remove it".format(apkfile))
 385                 else:
 386                     logging.error("Failed to get apk information, skipping " + apkfile)
 387                 continue
 388             for line in p.output.splitlines():
 389                 if line.startswith("package:"):
 390                     try:
 391                         thisinfo['id'] = re.match(name_pat, line).group(1)
 392                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 393                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 394                     except Exception, e:
 395                         logging.error("Package matching failed: " + str(e))
 396                         logging.info("Line was: " + line)
 397                         sys.exit(1)
 398                 elif line.startswith("application:"):
 399                     thisinfo['name'] = re.match(label_pat, line).group(1)
 400                     # Keep path to non-dpi icon in case we need it
 401                     match = re.match(icon_pat_nodpi, line)
 402                     if match:
 403                         thisinfo['icons_src']['-1'] = match.group(1)
 404                 elif line.startswith("launchable-activity:"):
 405                     # Only use launchable-activity as fallback to application
 406                     if not thisinfo['name']:
 407                         thisinfo['name'] = re.match(label_pat, line).group(1)
 408                     if '-1' not in thisinfo['icons_src']:
 409                         match = re.match(icon_pat_nodpi, line)
 410                         if match:
 411                             thisinfo['icons_src']['-1'] = match.group(1)
 412                 elif line.startswith("application-icon-"):
 413                     match = re.match(icon_pat, line)
 414                     if match:
 415                         density = match.group(1)
 416                         path = match.group(2)
 417                         thisinfo['icons_src'][density] = path
 418                 elif line.startswith("sdkVersion:"):
 419                     m = re.match(sdkversion_pat, line)
 420                     if m is None:
 421                         logging.error(line.replace('sdkVersion:', '')
 422                                       + ' is not a valid minSdkVersion!')
 423                     else:
 424                         thisinfo['sdkversion'] = m.group(1)
 425                 elif line.startswith("maxSdkVersion:"):
 426                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 427                 elif line.startswith("native-code:"):
 428                     thisinfo['nativecode'] = []
 429                     for arch in line[13:].split(' '):
 430                         thisinfo['nativecode'].append(arch[1:-1])
 431                 elif line.startswith("uses-permission:"):
 432                     perm = re.match(string_pat, line).group(1)
 433                     if perm.startswith("android.permission."):
 434                         perm = perm[19:]
 435                     thisinfo['permissions'].add(perm)
 436                 elif line.startswith("uses-feature:"):
 437                     perm = re.match(string_pat, line).group(1)
 438                     # Filter out this, it's only added with the latest SDK tools and
 439                     # causes problems for lots of apps.
 440                     if perm != "android.hardware.screen.portrait" \
 441                             and perm != "android.hardware.screen.landscape":
 442                         if perm.startswith("android.feature."):
 443                             perm = perm[16:]
 444                         thisinfo['features'].add(perm)
 445
 446             if 'sdkversion' not in thisinfo:
 447                 logging.warn("No SDK version information found in {0}".format(apkfile))
 448                 thisinfo['sdkversion'] = 0
 449
 450             # Check for debuggable apks...
 451             if common.isApkDebuggable(apkfile, config):
 452                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 453
 454             # Calculate the sha256...
 455             sha = hashlib.sha256()
 456             with open(apkfile, 'rb') as f:
 457                 while True:
 458                     t = f.read(1024)
 459                     if len(t) == 0:
 460                         break
 461                     sha.update(t)
 462                 thisinfo['sha256'] = sha.hexdigest()
 463
 464             # Get the signature (or md5 of, to be precise)...
 465             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 466             if not os.path.exists(getsig_dir + "/getsig.class"):
 467                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 468                 sys.exit(1)
 469             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 470                              'getsig', os.path.join(os.getcwd(), apkfile)])
 471             thisinfo['sig'] = None
 472             for line in p.output.splitlines():
 473                 if line.startswith('Result:'):
 474                     thisinfo['sig'] = line[7:].strip()
 475                     break
 476             if p.returncode != 0 or not thisinfo['sig']:
 477                 logging.critical("Failed to get apk signature")
 478                 sys.exit(1)
 479
 480             apk = zipfile.ZipFile(apkfile, 'r')
 481
 482             iconfilename = "%s.%s.png" % (
 483                 thisinfo['id'],
 484                 thisinfo['versioncode'])
 485
 486             # Extract the icon file...
 487             densities = get_densities()
 488             empty_densities = []
 489             for density in densities:
 490                 if density not in thisinfo['icons_src']:
 491                     empty_densities.append(density)
 492                     continue
 493                 iconsrc = thisinfo['icons_src'][density]
 494                 icon_dir = get_icon_dir(repodir, density)
 495                 icondest = os.path.join(icon_dir, iconfilename)
 496
 497                 try:
 498                     iconfile = open(icondest, 'wb')
 499                     iconfile.write(apk.read(iconsrc))
 500                     iconfile.close()
 501                     thisinfo['icons'][density] = iconfilename
 502
 503                 except:
 504                     logging.warn("Error retrieving icon file")
 505                     del thisinfo['icons'][density]
 506                     del thisinfo['icons_src'][density]
 507                     empty_densities.append(density)
 508
 509             if '-1' in thisinfo['icons_src']:
 510                 iconsrc = thisinfo['icons_src']['-1']
 511                 iconpath = os.path.join(
 512                     get_icon_dir(repodir, None), iconfilename)
 513                 iconfile = open(iconpath, 'wb')
 514                 iconfile.write(apk.read(iconsrc))
 515                 iconfile.close()
 516                 try:
 517                     im = Image.open(iconpath)
 518                     dpi = px_to_dpi(im.size[0])
 519                     for density in densities:
 520                         if density in thisinfo['icons']:
 521                             break
 522                         if density == densities[-1] or dpi >= int(density):
 523                             thisinfo['icons'][density] = iconfilename
 524                             shutil.move(iconpath,
 525                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 526                             empty_densities.remove(density)
 527                             break
 528                 except Exception, e:
 529                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 530
 531             if thisinfo['icons']:
 532                 thisinfo['icon'] = iconfilename
 533
 534             apk.close()
 535
 536             # First try resizing down to not lose quality
 537             last_density = None
 538             for density in densities:
 539                 if density not in empty_densities:
 540                     last_density = density
 541                     continue
 542                 if last_density is None:
 543                     continue
 544                 logging.debug("Density %s not available, resizing down from %s"
 545                               % (density, last_density))
 546
 547                 last_iconpath = os.path.join(
 548                     get_icon_dir(repodir, last_density), iconfilename)
 549                 iconpath = os.path.join(
 550                     get_icon_dir(repodir, density), iconfilename)
 551                 try:
 552                     im = Image.open(last_iconpath)
 553                 except:
 554                     logging.warn("Invalid image file at %s" % last_iconpath)
 555                     continue
 556
 557                 size = dpi_to_px(density)
 558
 559                 im.thumbnail((size, size), Image.ANTIALIAS)
 560                 im.save(iconpath, "PNG")
 561                 empty_densities.remove(density)
 562
 563             # Then just copy from the highest resolution available
 564             last_density = None
 565             for density in reversed(densities):
 566                 if density not in empty_densities:
 567                     last_density = density
 568                     continue
 569                 if last_density is None:
 570                     continue
 571                 logging.debug("Density %s not available, copying from lower density %s"
 572                               % (density, last_density))
 573
 574                 shutil.copyfile(
 575                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 576                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 577
 578                 empty_densities.remove(density)
 579
 580             for density in densities:
 581                 icon_dir = get_icon_dir(repodir, density)
 582                 icondest = os.path.join(icon_dir, iconfilename)
 583                 resize_icon(icondest, density)
 584
 585             # Copy from icons-mdpi to icons since mdpi is the baseline density
 586             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 587             if os.path.isfile(baseline):
 588                 shutil.copyfile(baseline,
 589                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 590
 591             # Record in known apks, getting the added date at the same time..
 592             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 593             if added:
 594                 thisinfo['added'] = added
 595
 596             apkcache[apkfilename] = thisinfo
 597             cachechanged = True
 598
 599         apks.append(thisinfo)
 600
 601     return apks, cachechanged
 602
 603
 604 repo_pubkey_fingerprint = None
 605
 606
 607 def make_index(apps, apks, repodir, archive, categories):
 608     """Make a repo index.
 609
 610     :param apps: fully populated apps list
 611     :param apks: full populated apks list
 612     :param repodir: the repo directory
 613     :param archive: True if this is the archive repo, False if it's the
 614                     main one.
 615     :param categories: list of categories
 616     """
 617
 618     doc = Document()
 619
 620     def addElement(name, value, doc, parent):
 621         el = doc.createElement(name)
 622         el.appendChild(doc.createTextNode(value))
 623         parent.appendChild(el)
 624
 625     def addElementCDATA(name, value, doc, parent):
 626         el = doc.createElement(name)
 627         el.appendChild(doc.createCDATASection(value))
 628         parent.appendChild(el)
 629
 630     root = doc.createElement("fdroid")
 631     doc.appendChild(root)
 632
 633     repoel = doc.createElement("repo")
 634
 635     if archive:
 636         repoel.setAttribute("name", config['archive_name'])
 637         if config['repo_maxage'] != 0:
 638             repoel.setAttribute("maxage", str(config['repo_maxage']))
 639         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 640         repoel.setAttribute("url", config['archive_url'])
 641         addElement('description', config['archive_description'], doc, repoel)
 642
 643     else:
 644         repoel.setAttribute("name", config['repo_name'])
 645         if config['repo_maxage'] != 0:
 646             repoel.setAttribute("maxage", str(config['repo_maxage']))
 647         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 648         repoel.setAttribute("url", config['repo_url'])
 649         addElement('description', config['repo_description'], doc, repoel)
 650
 651     repoel.setAttribute("version", "12")
 652     repoel.setAttribute("timestamp", str(int(time.time())))
 653
 654     if 'repo_keyalias' in config:
 655
 656         # Generate a certificate fingerprint the same way keytool does it
 657         # (but with slightly different formatting)
 658         def cert_fingerprint(data):
 659             digest = hashlib.sha256(data).digest()
 660             ret = []
 661             ret.append(' '.join("%02X" % ord(b) for b in digest))
 662             return " ".join(ret)
 663
 664         def extract_pubkey():
 665             p = FDroidPopen(['keytool', '-exportcert',
 666                              '-alias', config['repo_keyalias'],
 667                              '-keystore', config['keystore'],
 668                              '-storepass:file', config['keystorepassfile']]
 669                             + config['smartcardoptions'], output=False)
 670             if p.returncode != 0:
 671                 msg = "Failed to get repo pubkey!"
 672                 if config['keystore'] == 'NONE':
 673                     msg += ' Is your crypto smartcard plugged in?'
 674                 logging.critical(msg)
 675                 sys.exit(1)
 676             global repo_pubkey_fingerprint
 677             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 678             return "".join("%02x" % ord(b) for b in p.output)
 679
 680         repoel.setAttribute("pubkey", extract_pubkey())
 681
 682     root.appendChild(repoel)
 683
 684     for app in apps:
 685
 686         if app['Disabled'] is not None:
 687             continue
 688
 689         # Get a list of the apks for this app...
 690         apklist = []
 691         for apk in apks:
 692             if apk['id'] == app['id']:
 693                 apklist.append(apk)
 694
 695         if len(apklist) == 0:
 696             continue
 697
 698         apel = doc.createElement("application")
 699         apel.setAttribute("id", app['id'])
 700         root.appendChild(apel)
 701
 702         addElement('id', app['id'], doc, apel)
 703         if 'added' in app:
 704             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 705         if 'lastupdated' in app:
 706             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 707         addElement('name', app['Name'], doc, apel)
 708         addElement('summary', app['Summary'], doc, apel)
 709         if app['icon']:
 710             addElement('icon', app['icon'], doc, apel)
 711
 712         def linkres(link):
 713             for app in apps:
 714                 if app['id'] == link:
 715                     return ("fdroid.app:" + link, app['Name'])
 716             raise MetaDataException("Cannot resolve app id " + link)
 717         addElement('desc',
 718                    metadata.description_html(app['Description'], linkres),
 719                    doc, apel)
 720         addElement('license', app['License'], doc, apel)
 721         if 'Categories' in app:
 722             addElement('categories', ','.join(app["Categories"]), doc, apel)
 723             # We put the first (primary) category in LAST, which will have
 724             # the desired effect of making clients that only understand one
 725             # category see that one.
 726             addElement('category', app["Categories"][0], doc, apel)
 727         addElement('web', app['Web Site'], doc, apel)
 728         addElement('source', app['Source Code'], doc, apel)
 729         addElement('tracker', app['Issue Tracker'], doc, apel)
 730         if app['Donate']:
 731             addElement('donate', app['Donate'], doc, apel)
 732         if app['Bitcoin']:
 733             addElement('bitcoin', app['Bitcoin'], doc, apel)
 734         if app['Litecoin']:
 735             addElement('litecoin', app['Litecoin'], doc, apel)
 736         if app['Dogecoin']:
 737             addElement('dogecoin', app['Dogecoin'], doc, apel)
 738         if app['FlattrID']:
 739             addElement('flattr', app['FlattrID'], doc, apel)
 740
 741         # These elements actually refer to the current version (i.e. which
 742         # one is recommended. They are historically mis-named, and need
 743         # changing, but stay like this for now to support existing clients.
 744         addElement('marketversion', app['Current Version'], doc, apel)
 745         addElement('marketvercode', app['Current Version Code'], doc, apel)
 746
 747         if app['AntiFeatures']:
 748             af = app['AntiFeatures'].split(',')
 749             # TODO: Temporarily not including UpstreamNonFree in the index,
 750             # because current F-Droid clients do not understand it, and also
 751             # look ugly when they encounter an unknown antifeature. This
 752             # filtering can be removed in time...
 753             if 'UpstreamNonFree' in af:
 754                 af.remove('UpstreamNonFree')
 755             if af:
 756                 addElement('antifeatures', ','.join(af), doc, apel)
 757         if app['Provides']:
 758             pv = app['Provides'].split(',')
 759             addElement('provides', ','.join(pv), doc, apel)
 760         if app['Requires Root']:
 761             addElement('requirements', 'root', doc, apel)
 762
 763         # Sort the apk list into version order, just so the web site
 764         # doesn't have to do any work by default...
 765         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 766
 767         # Check for duplicates - they will make the client unhappy...
 768         for i in range(len(apklist) - 1):
 769             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 770                 logging.critical("duplicate versions: '%s' - '%s'" % (
 771                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 772                 sys.exit(1)
 773
 774         for apk in apklist:
 775             apkel = doc.createElement("package")
 776             apel.appendChild(apkel)
 777             addElement('version', apk['version'], doc, apkel)
 778             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 779             addElement('apkname', apk['apkname'], doc, apkel)
 780             if 'srcname' in apk:
 781                 addElement('srcname', apk['srcname'], doc, apkel)
 782             for hash_type in ['sha256']:
 783                 if hash_type not in apk:
 784                     continue
 785                 hashel = doc.createElement("hash")
 786                 hashel.setAttribute("type", hash_type)
 787                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 788                 apkel.appendChild(hashel)
 789             addElement('sig', apk['sig'], doc, apkel)
 790             addElement('size', str(apk['size']), doc, apkel)
 791             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 792             if 'maxsdkversion' in apk:
 793                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 794             if 'added' in apk:
 795                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 796             if app['Requires Root']:
 797                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 798                     apk['permissions'].add('ACCESS_SUPERUSER')
 799
 800             if len(apk['permissions']) > 0:
 801                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 802             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 803                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 804             if len(apk['features']) > 0:
 805                 addElement('features', ','.join(apk['features']), doc, apkel)
 806
 807     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 808     if options.pretty:
 809         output = doc.toprettyxml()
 810     else:
 811         output = doc.toxml()
 812     of.write(output)
 813     of.close()
 814
 815     if 'repo_keyalias' in config:
 816
 817         logging.info("Creating signed index with this key (SHA256):")
 818         logging.info("%s" % repo_pubkey_fingerprint)
 819
 820         # Create a jar of the index...
 821         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 822         if p.returncode != 0:
 823             logging.critical("Failed to create jar file")
 824             sys.exit(1)
 825
 826         # Sign the index...
 827         args = ['jarsigner', '-keystore', config['keystore'],
 828                 '-storepass:file', config['keystorepassfile'],
 829                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 830                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 831         if config['keystore'] == 'NONE':
 832             args += config['smartcardoptions']
 833         else:  # smardcards never use -keypass
 834             args += ['-keypass:file', config['keypassfile']]
 835         p = FDroidPopen(args)
 836         # TODO keypass should be sent via stdin
 837         if p.returncode != 0:
 838             logging.critical("Failed to sign index")
 839             sys.exit(1)
 840
 841     # Copy the repo icon into the repo directory...
 842     icon_dir = os.path.join(repodir, 'icons')
 843     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 844     shutil.copyfile(config['repo_icon'], iconfilename)
 845
 846     # Write a category list in the repo to allow quick access...
 847     catdata = ''
 848     for cat in categories:
 849         catdata += cat + '\n'
 850     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 851     f.write(catdata)
 852     f.close()
 853
 854
 855 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 856
 857     for app in apps:
 858
 859         # Get a list of the apks for this app...
 860         apklist = []
 861         for apk in apks:
 862             if apk['id'] == app['id']:
 863                 apklist.append(apk)
 864
 865         # Sort the apk list into version order...
 866         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 867
 868         if app['Archive Policy']:
 869             keepversions = int(app['Archive Policy'][:-9])
 870         else:
 871             keepversions = defaultkeepversions
 872
 873         if len(apklist) > keepversions:
 874             for apk in apklist[keepversions:]:
 875                 logging.info("Moving " + apk['apkname'] + " to archive")
 876                 shutil.move(os.path.join(repodir, apk['apkname']),
 877                             os.path.join(archivedir, apk['apkname']))
 878                 if 'srcname' in apk:
 879                     shutil.move(os.path.join(repodir, apk['srcname']),
 880                                 os.path.join(archivedir, apk['srcname']))
 881                     # Move GPG signature too...
 882                     sigfile = apk['srcname'] + '.asc'
 883                     sigsrc = os.path.join(repodir, sigfile)
 884                     if os.path.exists(sigsrc):
 885                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 886
 887                 archapks.append(apk)
 888                 apks.remove(apk)
 889
 890
 891 config = None
 892 options = None
 893
 894
 895 def main():
 896
 897     global config, options
 898
 899     # Parse command line...
 900     parser = OptionParser()
 901     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 902                       help="Create skeleton metadata files that are missing")
 903     parser.add_option("--delete-unknown", action="store_true", default=False,
 904                       help="Delete APKs without metadata from the repo")
 905     parser.add_option("-v", "--verbose", action="store_true", default=False,
 906                       help="Spew out even more information than normal")
 907     parser.add_option("-q", "--quiet", action="store_true", default=False,
 908                       help="Restrict output to warnings and errors")
 909     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 910                       help="Report on build data status")
 911     parser.add_option("-i", "--interactive", default=False, action="store_true",
 912                       help="Interactively ask about things that need updating.")
 913     parser.add_option("-I", "--icons", action="store_true", default=False,
 914                       help="Resize all the icons exceeding the max pixel size and exit")
 915     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 916                       help="Specify editor to use in interactive mode. Default " +
 917                       "is /etc/alternatives/editor")
 918     parser.add_option("-w", "--wiki", default=False, action="store_true",
 919                       help="Update the wiki")
 920     parser.add_option("", "--pretty", action="store_true", default=False,
 921                       help="Produce human-readable index.xml")
 922     parser.add_option("--clean", action="store_true", default=False,
 923                       help="Clean update - don't uses caches, reprocess all apks")
 924     (options, args) = parser.parse_args()
 925
 926     config = common.read_config(options)
 927
 928     repodirs = ['repo']
 929     if config['archive_older'] != 0:
 930         repodirs.append('archive')
 931         if not os.path.exists('archive'):
 932             os.mkdir('archive')
 933
 934     if options.icons:
 935         resize_all_icons(repodirs)
 936         sys.exit(0)
 937
 938     # check that icons exist now, rather than fail at the end of `fdroid update`
 939     for k in ['repo_icon', 'archive_icon']:
 940         if k in config:
 941             if not os.path.exists(config[k]):
 942                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 943                 sys.exit(1)
 944
 945     # Get all apps...
 946     apps = metadata.read_metadata()
 947
 948     # Generate a list of categories...
 949     categories = set()
 950     for app in apps:
 951         categories.update(app['Categories'])
 952
 953     # Read known apks data (will be updated and written back when we've finished)
 954     knownapks = common.KnownApks()
 955
 956     # Gather information about all the apk files in the repo directory, using
 957     # cached data if possible.
 958     apkcachefile = os.path.join('tmp', 'apkcache')
 959     if not options.clean and os.path.exists(apkcachefile):
 960         with open(apkcachefile, 'rb') as cf:
 961             apkcache = pickle.load(cf)
 962     else:
 963         apkcache = {}
 964     cachechanged = False
 965
 966     delete_disabled_builds(apps, apkcache, repodirs)
 967
 968     # Scan all apks in the main repo
 969     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 970     if cc:
 971         cachechanged = True
 972
 973     # Generate warnings for apk's with no metadata (or create skeleton
 974     # metadata files, if requested on the command line)
 975     newmetadata = False
 976     for apk in apks:
 977         found = False
 978         for app in apps:
 979             if app['id'] == apk['id']:
 980                 found = True
 981                 break
 982         if not found:
 983             if options.create_metadata:
 984                 if 'name' not in apk:
 985                     logging.error(apk['id'] + ' does not have a name! Skipping...')
 986                     continue
 987                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 988                 f.write("License:Unknown\n")
 989                 f.write("Web Site:\n")
 990                 f.write("Source Code:\n")
 991                 f.write("Issue Tracker:\n")
 992                 f.write("Summary:" + apk['name'] + "\n")
 993                 f.write("Description:\n")
 994                 f.write(apk['name'] + "\n")
 995                 f.write(".\n")
 996                 f.close()
 997                 logging.info("Generated skeleton metadata for " + apk['id'])
 998                 newmetadata = True
 999             else:
1000                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1001                 if options.delete_unknown:
1002                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1003                     rmf = os.path.join(repodirs[0], apk['apkname'])
1004                     if not os.path.exists(rmf):
1005                         logging.error("Could not find {0} to remove it".format(rmf))
1006                     else:
1007                         os.remove(rmf)
1008                 else:
1009                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1010
1011     # update the metadata with the newly created ones included
1012     if newmetadata:
1013         apps = metadata.read_metadata()
1014
1015     # Scan the archive repo for apks as well
1016     if len(repodirs) > 1:
1017         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1018         if cc:
1019             cachechanged = True
1020     else:
1021         archapks = []
1022
1023     # Some information from the apks needs to be applied up to the application
1024     # level. When doing this, we use the info from the most recent version's apk.
1025     # We deal with figuring out when the app was added and last updated at the
1026     # same time.
1027     for app in apps:
1028         bestver = 0
1029         added = None
1030         lastupdated = None
1031         for apk in apks + archapks:
1032             if apk['id'] == app['id']:
1033                 if apk['versioncode'] > bestver:
1034                     bestver = apk['versioncode']
1035                     bestapk = apk
1036
1037                 if 'added' in apk:
1038                     if not added or apk['added'] < added:
1039                         added = apk['added']
1040                     if not lastupdated or apk['added'] > lastupdated:
1041                         lastupdated = apk['added']
1042
1043         if added:
1044             app['added'] = added
1045         else:
1046             logging.warn("Don't know when " + app['id'] + " was added")
1047         if lastupdated:
1048             app['lastupdated'] = lastupdated
1049         else:
1050             logging.warn("Don't know when " + app['id'] + " was last updated")
1051
1052         if bestver == 0:
1053             if app['Name'] is None:
1054                 app['Name'] = app['id']
1055             app['icon'] = None
1056             logging.warn("Application " + app['id'] + " has no packages")
1057         else:
1058             if app['Name'] is None:
1059                 app['Name'] = bestapk['name']
1060             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1061
1062     # Sort the app list by name, then the web site doesn't have to by default.
1063     # (we had to wait until we'd scanned the apks to do this, because mostly the
1064     # name comes from there!)
1065     apps = sorted(apps, key=lambda app: app['Name'].upper())
1066
1067     if len(repodirs) > 1:
1068         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1069
1070     # Make the index for the main repo...
1071     make_index(apps, apks, repodirs[0], False, categories)
1072
1073     # If there's an archive repo,  make the index for it. We already scanned it
1074     # earlier on.
1075     if len(repodirs) > 1:
1076         make_index(apps, archapks, repodirs[1], True, categories)
1077
1078     if config['update_stats']:
1079
1080         # Update known apks info...
1081         knownapks.writeifchanged()
1082
1083         # Generate latest apps data for widget
1084         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1085             data = ''
1086             for line in file(os.path.join('stats', 'latestapps.txt')):
1087                 appid = line.rstrip()
1088                 data += appid + "\t"
1089                 for app in apps:
1090                     if app['id'] == appid:
1091                         data += app['Name'] + "\t"
1092                         if app['icon'] is not None:
1093                             data += app['icon'] + "\t"
1094                         data += app['License'] + "\n"
1095                         break
1096             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1097             f.write(data)
1098             f.close()
1099
1100     if cachechanged:
1101         with open(apkcachefile, 'wb') as cf:
1102             pickle.dump(apkcache, cf)
1103
1104     # Update the wiki...
1105     if options.wiki:
1106         update_wiki(apps, apks + archapks)
1107
1108     logging.info("Finished.")
1109
1110 if __name__ == "__main__":
1111     main()