fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen, SilentPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(app['id'], thisbuild['vercode'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304     except Exception, e:
 305         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 306
 307
 308 def resize_all_icons(repodirs):
 309     """Resize all icons that exceed the max size
 310
 311     :param repodirs: the repo directories to process
 312     """
 313     for repodir in repodirs:
 314         for density in get_densities():
 315             icon_dir = get_icon_dir(repodir, density)
 316             icon_glob = os.path.join(icon_dir, '*.png')
 317             for iconpath in glob.glob(icon_glob):
 318                 resize_icon(iconpath, density)
 319
 320
 321 def scan_apks(apps, apkcache, repodir, knownapks):
 322     """Scan the apks in the given repo directory.
 323
 324     This also extracts the icons.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodir: repo directory to scan
 329     :param knownapks: known apks info
 330     :returns: (apks, cachechanged) where apks is a list of apk information,
 331               and cachechanged is True if the apkcache got changed.
 332     """
 333
 334     cachechanged = False
 335
 336     icon_dirs = get_icon_dirs(repodir)
 337     for icon_dir in icon_dirs:
 338         if os.path.exists(icon_dir):
 339             if options.clean:
 340                 shutil.rmtree(icon_dir)
 341                 os.makedirs(icon_dir)
 342         else:
 343             os.makedirs(icon_dir)
 344
 345     apks = []
 346     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 347     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 348     vername_pat = re.compile(".*versionName='([^']*)'.*")
 349     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 350     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 351     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 352     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 353     string_pat = re.compile(".*'([^']*)'.*")
 354     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 355
 356         apkfilename = apkfile[len(repodir) + 1:]
 357         if ' ' in apkfilename:
 358             logging.critical("Spaces in filenames are not allowed.")
 359             sys.exit(1)
 360
 361         if apkfilename in apkcache:
 362             logging.debug("Reading " + apkfilename + " from cache")
 363             thisinfo = apkcache[apkfilename]
 364
 365         else:
 366             logging.debug("Processing " + apkfilename)
 367             thisinfo = {}
 368             thisinfo['apkname'] = apkfilename
 369             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 370             if os.path.exists(os.path.join(repodir, srcfilename)):
 371                 thisinfo['srcname'] = srcfilename
 372             thisinfo['size'] = os.path.getsize(apkfile)
 373             thisinfo['permissions'] = set()
 374             thisinfo['features'] = set()
 375             thisinfo['icons_src'] = {}
 376             thisinfo['icons'] = {}
 377             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 378             if p.returncode != 0:
 379                 if options.delete_unknown:
 380                     if os.path.exists(apkfile):
 381                         logging.error("Failed to get apk information, deleting " + apkfile)
 382                         os.remove(apkfile)
 383                     else:
 384                         logging.error("Could not find {0} to remove it".format(apkfile))
 385                 else:
 386                     logging.error("Failed to get apk information, skipping " + apkfile)
 387                 continue
 388             for line in p.output.splitlines():
 389                 if line.startswith("package:"):
 390                     try:
 391                         thisinfo['id'] = re.match(name_pat, line).group(1)
 392                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 393                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 394                     except Exception, e:
 395                         logging.error("Package matching failed: " + str(e))
 396                         logging.info("Line was: " + line)
 397                         sys.exit(1)
 398                 elif line.startswith("application:"):
 399                     thisinfo['name'] = re.match(label_pat, line).group(1)
 400                     # Keep path to non-dpi icon in case we need it
 401                     match = re.match(icon_pat_nodpi, line)
 402                     if match:
 403                         thisinfo['icons_src']['-1'] = match.group(1)
 404                 elif line.startswith("launchable-activity:"):
 405                     # Only use launchable-activity as fallback to application
 406                     if not thisinfo['name']:
 407                         thisinfo['name'] = re.match(label_pat, line).group(1)
 408                     if '-1' not in thisinfo['icons_src']:
 409                         match = re.match(icon_pat_nodpi, line)
 410                         if match:
 411                             thisinfo['icons_src']['-1'] = match.group(1)
 412                 elif line.startswith("application-icon-"):
 413                     match = re.match(icon_pat, line)
 414                     if match:
 415                         density = match.group(1)
 416                         path = match.group(2)
 417                         thisinfo['icons_src'][density] = path
 418                 elif line.startswith("sdkVersion:"):
 419                     m = re.match(sdkversion_pat, line)
 420                     if m is None:
 421                         logging.error(line.replace('sdkVersion:', '')
 422                                       + ' is not a valid minSdkVersion!')
 423                     else:
 424                         thisinfo['sdkversion'] = m.group(1)
 425                 elif line.startswith("maxSdkVersion:"):
 426                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 427                 elif line.startswith("native-code:"):
 428                     thisinfo['nativecode'] = []
 429                     for arch in line[13:].split(' '):
 430                         thisinfo['nativecode'].append(arch[1:-1])
 431                 elif line.startswith("uses-permission:"):
 432                     perm = re.match(string_pat, line).group(1)
 433                     if perm.startswith("android.permission."):
 434                         perm = perm[19:]
 435                     thisinfo['permissions'].add(perm)
 436                 elif line.startswith("uses-feature:"):
 437                     perm = re.match(string_pat, line).group(1)
 438                     # Filter out this, it's only added with the latest SDK tools and
 439                     # causes problems for lots of apps.
 440                     if perm != "android.hardware.screen.portrait" \
 441                             and perm != "android.hardware.screen.landscape":
 442                         if perm.startswith("android.feature."):
 443                             perm = perm[16:]
 444                         thisinfo['features'].add(perm)
 445
 446             if 'sdkversion' not in thisinfo:
 447                 logging.warn("No SDK version information found in {0}".format(apkfile))
 448                 thisinfo['sdkversion'] = 0
 449
 450             # Check for debuggable apks...
 451             if common.isApkDebuggable(apkfile, config):
 452                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 453
 454             # Calculate the sha256...
 455             sha = hashlib.sha256()
 456             with open(apkfile, 'rb') as f:
 457                 while True:
 458                     t = f.read(1024)
 459                     if len(t) == 0:
 460                         break
 461                     sha.update(t)
 462                 thisinfo['sha256'] = sha.hexdigest()
 463
 464             # Get the signature (or md5 of, to be precise)...
 465             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 466             if not os.path.exists(getsig_dir + "/getsig.class"):
 467                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 468                 sys.exit(1)
 469             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 470                              'getsig', os.path.join(os.getcwd(), apkfile)])
 471             if p.returncode != 0 or not p.output.startswith('Result:'):
 472                 logging.critical("Failed to get apk signature")
 473                 sys.exit(1)
 474             thisinfo['sig'] = p.output[7:].strip()
 475
 476             apk = zipfile.ZipFile(apkfile, 'r')
 477
 478             iconfilename = "%s.%s.png" % (
 479                 thisinfo['id'],
 480                 thisinfo['versioncode'])
 481
 482             # Extract the icon file...
 483             densities = get_densities()
 484             empty_densities = []
 485             for density in densities:
 486                 if density not in thisinfo['icons_src']:
 487                     empty_densities.append(density)
 488                     continue
 489                 iconsrc = thisinfo['icons_src'][density]
 490                 icon_dir = get_icon_dir(repodir, density)
 491                 icondest = os.path.join(icon_dir, iconfilename)
 492
 493                 try:
 494                     iconfile = open(icondest, 'wb')
 495                     iconfile.write(apk.read(iconsrc))
 496                     iconfile.close()
 497                     thisinfo['icons'][density] = iconfilename
 498
 499                 except:
 500                     logging.warn("Error retrieving icon file")
 501                     del thisinfo['icons'][density]
 502                     del thisinfo['icons_src'][density]
 503                     empty_densities.append(density)
 504
 505             if '-1' in thisinfo['icons_src']:
 506                 iconsrc = thisinfo['icons_src']['-1']
 507                 iconpath = os.path.join(
 508                     get_icon_dir(repodir, None), iconfilename)
 509                 iconfile = open(iconpath, 'wb')
 510                 iconfile.write(apk.read(iconsrc))
 511                 iconfile.close()
 512                 try:
 513                     im = Image.open(iconpath)
 514                     dpi = px_to_dpi(im.size[0])
 515                     for density in densities:
 516                         if density in thisinfo['icons']:
 517                             break
 518                         if density == densities[-1] or dpi >= int(density):
 519                             thisinfo['icons'][density] = iconfilename
 520                             shutil.move(iconpath,
 521                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 522                             empty_densities.remove(density)
 523                             break
 524                 except Exception, e:
 525                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 526
 527             if thisinfo['icons']:
 528                 thisinfo['icon'] = iconfilename
 529
 530             apk.close()
 531
 532             # First try resizing down to not lose quality
 533             last_density = None
 534             for density in densities:
 535                 if density not in empty_densities:
 536                     last_density = density
 537                     continue
 538                 if last_density is None:
 539                     continue
 540                 logging.debug("Density %s not available, resizing down from %s"
 541                               % (density, last_density))
 542
 543                 last_iconpath = os.path.join(
 544                     get_icon_dir(repodir, last_density), iconfilename)
 545                 iconpath = os.path.join(
 546                     get_icon_dir(repodir, density), iconfilename)
 547                 try:
 548                     im = Image.open(last_iconpath)
 549                 except:
 550                     logging.warn("Invalid image file at %s" % last_iconpath)
 551                     continue
 552
 553                 size = dpi_to_px(density)
 554
 555                 im.thumbnail((size, size), Image.ANTIALIAS)
 556                 im.save(iconpath, "PNG")
 557                 empty_densities.remove(density)
 558
 559             # Then just copy from the highest resolution available
 560             last_density = None
 561             for density in reversed(densities):
 562                 if density not in empty_densities:
 563                     last_density = density
 564                     continue
 565                 if last_density is None:
 566                     continue
 567                 logging.debug("Density %s not available, copying from lower density %s"
 568                               % (density, last_density))
 569
 570                 shutil.copyfile(
 571                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 572                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 573
 574                 empty_densities.remove(density)
 575
 576             for density in densities:
 577                 icon_dir = get_icon_dir(repodir, density)
 578                 icondest = os.path.join(icon_dir, iconfilename)
 579                 resize_icon(icondest, density)
 580
 581             # Copy from icons-mdpi to icons since mdpi is the baseline density
 582             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 583             if os.path.isfile(baseline):
 584                 shutil.copyfile(baseline,
 585                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 586
 587             # Record in known apks, getting the added date at the same time..
 588             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 589             if added:
 590                 thisinfo['added'] = added
 591
 592             apkcache[apkfilename] = thisinfo
 593             cachechanged = True
 594
 595         apks.append(thisinfo)
 596
 597     return apks, cachechanged
 598
 599
 600 repo_pubkey_fingerprint = None
 601
 602
 603 def make_index(apps, apks, repodir, archive, categories):
 604     """Make a repo index.
 605
 606     :param apps: fully populated apps list
 607     :param apks: full populated apks list
 608     :param repodir: the repo directory
 609     :param archive: True if this is the archive repo, False if it's the
 610                     main one.
 611     :param categories: list of categories
 612     """
 613
 614     doc = Document()
 615
 616     def addElement(name, value, doc, parent):
 617         el = doc.createElement(name)
 618         el.appendChild(doc.createTextNode(value))
 619         parent.appendChild(el)
 620
 621     def addElementCDATA(name, value, doc, parent):
 622         el = doc.createElement(name)
 623         el.appendChild(doc.createCDATASection(value))
 624         parent.appendChild(el)
 625
 626     root = doc.createElement("fdroid")
 627     doc.appendChild(root)
 628
 629     repoel = doc.createElement("repo")
 630
 631     if archive:
 632         repoel.setAttribute("name", config['archive_name'])
 633         if config['repo_maxage'] != 0:
 634             repoel.setAttribute("maxage", str(config['repo_maxage']))
 635         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 636         repoel.setAttribute("url", config['archive_url'])
 637         addElement('description', config['archive_description'], doc, repoel)
 638
 639     else:
 640         repoel.setAttribute("name", config['repo_name'])
 641         if config['repo_maxage'] != 0:
 642             repoel.setAttribute("maxage", str(config['repo_maxage']))
 643         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 644         repoel.setAttribute("url", config['repo_url'])
 645         addElement('description', config['repo_description'], doc, repoel)
 646
 647     repoel.setAttribute("version", "12")
 648     repoel.setAttribute("timestamp", str(int(time.time())))
 649
 650     if 'repo_keyalias' in config:
 651
 652         # Generate a certificate fingerprint the same way keytool does it
 653         # (but with slightly different formatting)
 654         def cert_fingerprint(data):
 655             digest = hashlib.sha256(data).digest()
 656             ret = []
 657             ret.append(' '.join("%02X" % ord(b) for b in digest))
 658             return " ".join(ret)
 659
 660         def extract_pubkey():
 661             p = FDroidPopen(['keytool', '-exportcert',
 662                              '-alias', config['repo_keyalias'],
 663                              '-keystore', config['keystore'],
 664                              '-storepass:file', config['keystorepassfile']]
 665                             + config['smartcardoptions'], output=False)
 666             if p.returncode != 0:
 667                 msg = "Failed to get repo pubkey!"
 668                 if config['keystore'] == 'NONE':
 669                     msg += ' Is your crypto smartcard plugged in?'
 670                 logging.critical(msg)
 671                 sys.exit(1)
 672             global repo_pubkey_fingerprint
 673             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 674             return "".join("%02x" % ord(b) for b in p.output)
 675
 676         repoel.setAttribute("pubkey", extract_pubkey())
 677
 678     root.appendChild(repoel)
 679
 680     for app in apps:
 681
 682         if app['Disabled'] is not None:
 683             continue
 684
 685         # Get a list of the apks for this app...
 686         apklist = []
 687         for apk in apks:
 688             if apk['id'] == app['id']:
 689                 apklist.append(apk)
 690
 691         if len(apklist) == 0:
 692             continue
 693
 694         apel = doc.createElement("application")
 695         apel.setAttribute("id", app['id'])
 696         root.appendChild(apel)
 697
 698         addElement('id', app['id'], doc, apel)
 699         if 'added' in app:
 700             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 701         if 'lastupdated' in app:
 702             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 703         addElement('name', app['Name'], doc, apel)
 704         addElement('summary', app['Summary'], doc, apel)
 705         if app['icon']:
 706             addElement('icon', app['icon'], doc, apel)
 707
 708         def linkres(link):
 709             for app in apps:
 710                 if app['id'] == link:
 711                     return ("fdroid.app:" + link, app['Name'])
 712             raise MetaDataException("Cannot resolve app id " + link)
 713         addElement('desc',
 714                    metadata.description_html(app['Description'], linkres),
 715                    doc, apel)
 716         addElement('license', app['License'], doc, apel)
 717         if 'Categories' in app:
 718             addElement('categories', ','.join(app["Categories"]), doc, apel)
 719             # We put the first (primary) category in LAST, which will have
 720             # the desired effect of making clients that only understand one
 721             # category see that one.
 722             addElement('category', app["Categories"][0], doc, apel)
 723         addElement('web', app['Web Site'], doc, apel)
 724         addElement('source', app['Source Code'], doc, apel)
 725         addElement('tracker', app['Issue Tracker'], doc, apel)
 726         if app['Donate']:
 727             addElement('donate', app['Donate'], doc, apel)
 728         if app['Bitcoin']:
 729             addElement('bitcoin', app['Bitcoin'], doc, apel)
 730         if app['Litecoin']:
 731             addElement('litecoin', app['Litecoin'], doc, apel)
 732         if app['Dogecoin']:
 733             addElement('dogecoin', app['Dogecoin'], doc, apel)
 734         if app['FlattrID']:
 735             addElement('flattr', app['FlattrID'], doc, apel)
 736
 737         # These elements actually refer to the current version (i.e. which
 738         # one is recommended. They are historically mis-named, and need
 739         # changing, but stay like this for now to support existing clients.
 740         addElement('marketversion', app['Current Version'], doc, apel)
 741         addElement('marketvercode', app['Current Version Code'], doc, apel)
 742
 743         if app['AntiFeatures']:
 744             af = app['AntiFeatures'].split(',')
 745             # TODO: Temporarily not including UpstreamNonFree in the index,
 746             # because current F-Droid clients do not understand it, and also
 747             # look ugly when they encounter an unknown antifeature. This
 748             # filtering can be removed in time...
 749             if 'UpstreamNonFree' in af:
 750                 af.remove('UpstreamNonFree')
 751             if af:
 752                 addElement('antifeatures', ','.join(af), doc, apel)
 753         if app['Provides']:
 754             pv = app['Provides'].split(',')
 755             addElement('provides', ','.join(pv), doc, apel)
 756         if app['Requires Root']:
 757             addElement('requirements', 'root', doc, apel)
 758
 759         # Sort the apk list into version order, just so the web site
 760         # doesn't have to do any work by default...
 761         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 762
 763         # Check for duplicates - they will make the client unhappy...
 764         for i in range(len(apklist) - 1):
 765             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 766                 logging.critical("duplicate versions: '%s' - '%s'" % (
 767                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 768                 sys.exit(1)
 769
 770         for apk in apklist:
 771             apkel = doc.createElement("package")
 772             apel.appendChild(apkel)
 773             addElement('version', apk['version'], doc, apkel)
 774             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 775             addElement('apkname', apk['apkname'], doc, apkel)
 776             if 'srcname' in apk:
 777                 addElement('srcname', apk['srcname'], doc, apkel)
 778             for hash_type in ['sha256']:
 779                 if hash_type not in apk:
 780                     continue
 781                 hashel = doc.createElement("hash")
 782                 hashel.setAttribute("type", hash_type)
 783                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 784                 apkel.appendChild(hashel)
 785             addElement('sig', apk['sig'], doc, apkel)
 786             addElement('size', str(apk['size']), doc, apkel)
 787             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 788             if 'maxsdkversion' in apk:
 789                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 790             if 'added' in apk:
 791                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 792             if app['Requires Root']:
 793                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 794                     apk['permissions'].append('ACCESS_SUPERUSER')
 795
 796             if len(apk['permissions']) > 0:
 797                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 798             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 799                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 800             if len(apk['features']) > 0:
 801                 addElement('features', ','.join(apk['features']), doc, apkel)
 802
 803     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 804     if options.pretty:
 805         output = doc.toprettyxml()
 806     else:
 807         output = doc.toxml()
 808     of.write(output)
 809     of.close()
 810
 811     if 'repo_keyalias' in config:
 812
 813         logging.info("Creating signed index with this key (SHA256):")
 814         logging.info("%s" % repo_pubkey_fingerprint)
 815
 816         # Create a jar of the index...
 817         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 818         if p.returncode != 0:
 819             logging.critical("Failed to create jar file")
 820             sys.exit(1)
 821
 822         # Sign the index...
 823         args = ['jarsigner', '-keystore', config['keystore'],
 824                 '-storepass:file', config['keystorepassfile'],
 825                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 826                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 827         if config['keystore'] == 'NONE':
 828             args += config['smartcardoptions']
 829         else:  # smardcards never use -keypass
 830             args += ['-keypass:file', config['keypassfile']]
 831         p = FDroidPopen(args)
 832         # TODO keypass should be sent via stdin
 833         if p.returncode != 0:
 834             logging.critical("Failed to sign index")
 835             sys.exit(1)
 836
 837     # Copy the repo icon into the repo directory...
 838     icon_dir = os.path.join(repodir, 'icons')
 839     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 840     shutil.copyfile(config['repo_icon'], iconfilename)
 841
 842     # Write a category list in the repo to allow quick access...
 843     catdata = ''
 844     for cat in categories:
 845         catdata += cat + '\n'
 846     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 847     f.write(catdata)
 848     f.close()
 849
 850
 851 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 852
 853     for app in apps:
 854
 855         # Get a list of the apks for this app...
 856         apklist = []
 857         for apk in apks:
 858             if apk['id'] == app['id']:
 859                 apklist.append(apk)
 860
 861         # Sort the apk list into version order...
 862         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 863
 864         if app['Archive Policy']:
 865             keepversions = int(app['Archive Policy'][:-9])
 866         else:
 867             keepversions = defaultkeepversions
 868
 869         if len(apklist) > keepversions:
 870             for apk in apklist[keepversions:]:
 871                 logging.info("Moving " + apk['apkname'] + " to archive")
 872                 shutil.move(os.path.join(repodir, apk['apkname']),
 873                             os.path.join(archivedir, apk['apkname']))
 874                 if 'srcname' in apk:
 875                     shutil.move(os.path.join(repodir, apk['srcname']),
 876                                 os.path.join(archivedir, apk['srcname']))
 877                     # Move GPG signature too...
 878                     sigfile = apk['srcname'] + '.asc'
 879                     sigsrc = os.path.join(repodir, sigfile)
 880                     if os.path.exists(sigsrc):
 881                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 882
 883                 archapks.append(apk)
 884                 apks.remove(apk)
 885
 886
 887 config = None
 888 options = None
 889
 890
 891 def main():
 892
 893     global config, options
 894
 895     # Parse command line...
 896     parser = OptionParser()
 897     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 898                       help="Create skeleton metadata files that are missing")
 899     parser.add_option("--delete-unknown", action="store_true", default=False,
 900                       help="Delete APKs without metadata from the repo")
 901     parser.add_option("-v", "--verbose", action="store_true", default=False,
 902                       help="Spew out even more information than normal")
 903     parser.add_option("-q", "--quiet", action="store_true", default=False,
 904                       help="Restrict output to warnings and errors")
 905     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 906                       help="Report on build data status")
 907     parser.add_option("-i", "--interactive", default=False, action="store_true",
 908                       help="Interactively ask about things that need updating.")
 909     parser.add_option("-I", "--icons", action="store_true", default=False,
 910                       help="Resize all the icons exceeding the max pixel size and exit")
 911     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 912                       help="Specify editor to use in interactive mode. Default " +
 913                       "is /etc/alternatives/editor")
 914     parser.add_option("-w", "--wiki", default=False, action="store_true",
 915                       help="Update the wiki")
 916     parser.add_option("", "--pretty", action="store_true", default=False,
 917                       help="Produce human-readable index.xml")
 918     parser.add_option("--clean", action="store_true", default=False,
 919                       help="Clean update - don't uses caches, reprocess all apks")
 920     (options, args) = parser.parse_args()
 921
 922     config = common.read_config(options)
 923
 924     repodirs = ['repo']
 925     if config['archive_older'] != 0:
 926         repodirs.append('archive')
 927         if not os.path.exists('archive'):
 928             os.mkdir('archive')
 929
 930     if options.icons:
 931         resize_all_icons(repodirs)
 932         sys.exit(0)
 933
 934     # check that icons exist now, rather than fail at the end of `fdroid update`
 935     for k in ['repo_icon', 'archive_icon']:
 936         if k in config:
 937             if not os.path.exists(config[k]):
 938                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 939                 sys.exit(1)
 940
 941     # Get all apps...
 942     apps = metadata.read_metadata()
 943
 944     # Generate a list of categories...
 945     categories = set()
 946     for app in apps:
 947         categories.update(app['Categories'])
 948
 949     # Read known apks data (will be updated and written back when we've finished)
 950     knownapks = common.KnownApks()
 951
 952     # Gather information about all the apk files in the repo directory, using
 953     # cached data if possible.
 954     apkcachefile = os.path.join('tmp', 'apkcache')
 955     if not options.clean and os.path.exists(apkcachefile):
 956         with open(apkcachefile, 'rb') as cf:
 957             apkcache = pickle.load(cf)
 958     else:
 959         apkcache = {}
 960     cachechanged = False
 961
 962     delete_disabled_builds(apps, apkcache, repodirs)
 963
 964     # Scan all apks in the main repo
 965     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 966     if cc:
 967         cachechanged = True
 968
 969     # Generate warnings for apk's with no metadata (or create skeleton
 970     # metadata files, if requested on the command line)
 971     newmetadata = False
 972     for apk in apks:
 973         found = False
 974         for app in apps:
 975             if app['id'] == apk['id']:
 976                 found = True
 977                 break
 978         if not found:
 979             if options.create_metadata:
 980                 if 'name' not in apk:
 981                     logging.error(apk['id'] + ' does not have a name! Skipping...')
 982                     continue
 983                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 984                 f.write("License:Unknown\n")
 985                 f.write("Web Site:\n")
 986                 f.write("Source Code:\n")
 987                 f.write("Issue Tracker:\n")
 988                 f.write("Summary:" + apk['name'] + "\n")
 989                 f.write("Description:\n")
 990                 f.write(apk['name'] + "\n")
 991                 f.write(".\n")
 992                 f.close()
 993                 logging.info("Generated skeleton metadata for " + apk['id'])
 994                 newmetadata = True
 995             else:
 996                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
 997                 if options.delete_unknown:
 998                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
 999                     rmf = os.path.join(repodirs[0], apk['apkname'])
1000                     if not os.path.exists(rmf):
1001                         logging.error("Could not find {0} to remove it".format(rmf))
1002                     else:
1003                         os.remove(rmf)
1004                 else:
1005                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1006
1007     # update the metadata with the newly created ones included
1008     if newmetadata:
1009         apps = metadata.read_metadata()
1010
1011     # Scan the archive repo for apks as well
1012     if len(repodirs) > 1:
1013         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1014         if cc:
1015             cachechanged = True
1016     else:
1017         archapks = []
1018
1019     # Some information from the apks needs to be applied up to the application
1020     # level. When doing this, we use the info from the most recent version's apk.
1021     # We deal with figuring out when the app was added and last updated at the
1022     # same time.
1023     for app in apps:
1024         bestver = 0
1025         added = None
1026         lastupdated = None
1027         for apk in apks + archapks:
1028             if apk['id'] == app['id']:
1029                 if apk['versioncode'] > bestver:
1030                     bestver = apk['versioncode']
1031                     bestapk = apk
1032
1033                 if 'added' in apk:
1034                     if not added or apk['added'] < added:
1035                         added = apk['added']
1036                     if not lastupdated or apk['added'] > lastupdated:
1037                         lastupdated = apk['added']
1038
1039         if added:
1040             app['added'] = added
1041         else:
1042             logging.warn("Don't know when " + app['id'] + " was added")
1043         if lastupdated:
1044             app['lastupdated'] = lastupdated
1045         else:
1046             logging.warn("Don't know when " + app['id'] + " was last updated")
1047
1048         if bestver == 0:
1049             if app['Name'] is None:
1050                 app['Name'] = app['id']
1051             app['icon'] = None
1052             logging.warn("Application " + app['id'] + " has no packages")
1053         else:
1054             if app['Name'] is None:
1055                 app['Name'] = bestapk['name']
1056             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1057
1058     # Sort the app list by name, then the web site doesn't have to by default.
1059     # (we had to wait until we'd scanned the apks to do this, because mostly the
1060     # name comes from there!)
1061     apps = sorted(apps, key=lambda app: app['Name'].upper())
1062
1063     if len(repodirs) > 1:
1064         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1065
1066     # Make the index for the main repo...
1067     make_index(apps, apks, repodirs[0], False, categories)
1068
1069     # If there's an archive repo,  make the index for it. We already scanned it
1070     # earlier on.
1071     if len(repodirs) > 1:
1072         make_index(apps, archapks, repodirs[1], True, categories)
1073
1074     if config['update_stats']:
1075
1076         # Update known apks info...
1077         knownapks.writeifchanged()
1078
1079         # Generate latest apps data for widget
1080         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1081             data = ''
1082             for line in file(os.path.join('stats', 'latestapps.txt')):
1083                 appid = line.rstrip()
1084                 data += appid + "\t"
1085                 for app in apps:
1086                     if app['id'] == appid:
1087                         data += app['Name'] + "\t"
1088                         if app['icon'] is not None:
1089                             data += app['icon'] + "\t"
1090                         data += app['License'] + "\n"
1091                         break
1092             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1093             f.write(data)
1094             f.close()
1095
1096     if cachechanged:
1097         with open(apkcachefile, 'wb') as cf:
1098             pickle.dump(apkcache, cf)
1099
1100     # Update the wiki...
1101     if options.wiki:
1102         update_wiki(apps, apks + archapks)
1103
1104     logging.info("Finished.")
1105
1106 if __name__ == "__main__":
1107     main()