fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen, SilentPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, sortedids, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80
  81     for appid in sortedids:
  82         app = apps[appid]
  83
  84         wikidata = ''
  85         if app['Disabled']:
  86             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  87         if app['AntiFeatures']:
  88             for af in app['AntiFeatures'].split(','):
  89                 wikidata += '{{AntiFeature|' + af + '}}\n'
  90         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  91             appid,
  92             app['Name'],
  93             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  94             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  95             app['Source Code'],
  96             app['Issue Tracker'],
  97             app['Web Site'],
  98             app['Donate'],
  99             app['FlattrID'],
 100             app['Bitcoin'],
 101             app['Litecoin'],
 102             app['Dogecoin'],
 103             app['License'],
 104             app.get('Requires Root', 'No'))
 105
 106         if app['Provides']:
 107             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 108
 109         wikidata += app['Summary']
 110         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 111
 112         wikidata += "=Description=\n"
 113         wikidata += metadata.description_wiki(app['Description']) + "\n"
 114
 115         wikidata += "=Maintainer Notes=\n"
 116         if 'Maintainer Notes' in app:
 117             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 118         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 119
 120         # Get a list of all packages for this application...
 121         apklist = []
 122         gotcurrentver = False
 123         cantupdate = False
 124         buildfails = False
 125         for apk in apks:
 126             if apk['id'] == appid:
 127                 if str(apk['versioncode']) == app['Current Version Code']:
 128                     gotcurrentver = True
 129                 apklist.append(apk)
 130         # Include ones we can't build, as a special case...
 131         for thisbuild in app['builds']:
 132             if thisbuild['disable']:
 133                 if thisbuild['vercode'] == app['Current Version Code']:
 134                     cantupdate = True
 135                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 136                 apklist.append({'versioncode': int(thisbuild['vercode']),
 137                                 'version': thisbuild['version'],
 138                                 'buildproblem': thisbuild['disable']
 139                                 })
 140             else:
 141                 builtit = False
 142                 for apk in apklist:
 143                     if apk['versioncode'] == int(thisbuild['vercode']):
 144                         builtit = True
 145                         break
 146                 if not builtit:
 147                     buildfails = True
 148                     apklist.append({'versioncode': int(thisbuild['vercode']),
 149                                     'version': thisbuild['version'],
 150                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 151                                     })
 152         if app['Current Version Code'] == '0':
 153             cantupdate = True
 154         # Sort with most recent first...
 155         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 156
 157         wikidata += "=Versions=\n"
 158         if len(apklist) == 0:
 159             wikidata += "We currently have no versions of this app available."
 160         elif not gotcurrentver:
 161             wikidata += "We don't have the current version of this app."
 162         else:
 163             wikidata += "We have the current version of this app."
 164         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 165         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 166         if len(app['No Source Since']) > 0:
 167             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 168         if len(app['Current Version']) > 0:
 169             wikidata += "The current (recommended) version is " + app['Current Version']
 170             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 171         validapks = 0
 172         for apk in apklist:
 173             wikidata += "==" + apk['version'] + "==\n"
 174
 175             if 'buildproblem' in apk:
 176                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 177             else:
 178                 validapks += 1
 179                 wikidata += "This version is built and signed by "
 180                 if 'srcname' in apk:
 181                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 182                 else:
 183                     wikidata += "the original developer.\n\n"
 184             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 185
 186         wikidata += '\n[[Category:' + wikicat + ']]\n'
 187         if len(app['No Source Since']) > 0:
 188             wikidata += '\n[[Category:Apps missing source code]]\n'
 189         if validapks == 0 and not app['Disabled']:
 190             wikidata += '\n[[Category:Apps with no packages]]\n'
 191         if cantupdate and not app['Disabled']:
 192             wikidata += "\n[[Category:Apps we can't update]]\n"
 193         if buildfails and not app['Disabled']:
 194             wikidata += "\n[[Category:Apps with failing builds]]\n"
 195         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 196             wikidata += '\n[[Category:Apps to Update]]\n'
 197         if app['Disabled']:
 198             wikidata += '\n[[Category:Apps that are disabled]]\n'
 199         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 200             wikidata += '\n[[Category:Apps with no update check]]\n'
 201         for appcat in app['Categories']:
 202             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 203
 204         # We can't have underscores in the page name, even if they're in
 205         # the package ID, because MediaWiki messes with them...
 206         pagename = appid.replace('_', ' ')
 207
 208         # Drop a trailing newline, because mediawiki is going to drop it anyway
 209         # and it we don't we'll think the page has changed when it hasn't...
 210         if wikidata.endswith('\n'):
 211             wikidata = wikidata[:-1]
 212
 213         generated_pages[pagename] = wikidata
 214
 215         # Make a redirect from the name to the ID too, unless there's
 216         # already an existing page with the name and it isn't a redirect.
 217         noclobber = False
 218         apppagename = app['Name'].replace('_', ' ')
 219         apppagename = apppagename.replace('{', '')
 220         apppagename = apppagename.replace('}', ' ')
 221         apppagename = apppagename.replace(':', ' ')
 222         # Drop double spaces caused mostly by replacing ':' above
 223         apppagename = apppagename.replace('  ', ' ')
 224         for expagename in site.allpages(prefix=apppagename,
 225                                         filterredir='nonredirects',
 226                                         generator=False):
 227             if expagename == apppagename:
 228                 noclobber = True
 229         # Another reason not to make the redirect page is if the app name
 230         # is the same as it's ID, because that will overwrite the real page
 231         # with an redirect to itself! (Although it seems like an odd
 232         # scenario this happens a lot, e.g. where there is metadata but no
 233         # builds or binaries to extract a name from.
 234         if apppagename == pagename:
 235             noclobber = True
 236         if not noclobber:
 237             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 238
 239     for tcat, genp in [(wikicat, generated_pages),
 240                        (wikiredircat, generated_redirects)]:
 241         catpages = site.Pages['Category:' + tcat]
 242         existingpages = []
 243         for page in catpages:
 244             existingpages.append(page.name)
 245             if page.name in genp:
 246                 pagetxt = page.edit()
 247                 if pagetxt != genp[page.name]:
 248                     logging.debug("Updating modified page " + page.name)
 249                     page.save(genp[page.name], summary='Auto-updated')
 250                 else:
 251                     logging.debug("Page " + page.name + " is unchanged")
 252             else:
 253                 logging.warn("Deleting page " + page.name)
 254                 page.delete('No longer published')
 255         for pagename, text in genp.items():
 256             logging.debug("Checking " + pagename)
 257             if pagename not in existingpages:
 258                 logging.debug("Creating page " + pagename)
 259                 try:
 260                     newpage = site.Pages[pagename]
 261                     newpage.save(text, summary='Auto-created')
 262                 except:
 263                     logging.error("...FAILED to create page")
 264
 265     # Purge server cache to ensure counts are up to date
 266     site.pages['Repository Maintenance'].purge()
 267
 268
 269 def delete_disabled_builds(apps, apkcache, repodirs):
 270     """Delete disabled build outputs.
 271
 272     :param apps: list of all applications, as per metadata.read_metadata
 273     :param apkcache: current apk cache information
 274     :param repodirs: the repo directories to process
 275     """
 276     for appid, app in apps.iteritems():
 277         for build in app['builds']:
 278             if build['disable']:
 279                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 280                 for repodir in repodirs:
 281                     apkpath = os.path.join(repodir, apkfilename)
 282                     ascpath = apkpath + ".asc"
 283                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 284                     for name in [apkpath, srcpath, ascpath]:
 285                         if os.path.exists(name):
 286                             logging.warn("Deleting disabled build output " + apkfilename)
 287                             os.remove(name)
 288                 if apkfilename in apkcache:
 289                     del apkcache[apkfilename]
 290
 291
 292 def resize_icon(iconpath, density):
 293
 294     if not os.path.isfile(iconpath):
 295         return
 296
 297     try:
 298         im = Image.open(iconpath)
 299         size = dpi_to_px(density)
 300
 301         if any(length > size for length in im.size):
 302             oldsize = im.size
 303             im.thumbnail((size, size), Image.ANTIALIAS)
 304             logging.debug("%s was too large at %s - new size is %s" % (
 305                 iconpath, oldsize, im.size))
 306             im.save(iconpath, "PNG")
 307
 308     except Exception, e:
 309         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 310
 311
 312 def resize_all_icons(repodirs):
 313     """Resize all icons that exceed the max size
 314
 315     :param repodirs: the repo directories to process
 316     """
 317     for repodir in repodirs:
 318         for density in get_densities():
 319             icon_dir = get_icon_dir(repodir, density)
 320             icon_glob = os.path.join(icon_dir, '*.png')
 321             for iconpath in glob.glob(icon_glob):
 322                 resize_icon(iconpath, density)
 323
 324
 325 def scan_apks(apps, apkcache, repodir, knownapks):
 326     """Scan the apks in the given repo directory.
 327
 328     This also extracts the icons.
 329
 330     :param apps: list of all applications, as per metadata.read_metadata
 331     :param apkcache: current apk cache information
 332     :param repodir: repo directory to scan
 333     :param knownapks: known apks info
 334     :returns: (apks, cachechanged) where apks is a list of apk information,
 335               and cachechanged is True if the apkcache got changed.
 336     """
 337
 338     cachechanged = False
 339
 340     icon_dirs = get_icon_dirs(repodir)
 341     for icon_dir in icon_dirs:
 342         if os.path.exists(icon_dir):
 343             if options.clean:
 344                 shutil.rmtree(icon_dir)
 345                 os.makedirs(icon_dir)
 346         else:
 347             os.makedirs(icon_dir)
 348
 349     apks = []
 350     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 351     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 352     vername_pat = re.compile(".*versionName='([^']*)'.*")
 353     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 354     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 355     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 356     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 357     string_pat = re.compile(".*'([^']*)'.*")
 358     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 359
 360         apkfilename = apkfile[len(repodir) + 1:]
 361         if ' ' in apkfilename:
 362             logging.critical("Spaces in filenames are not allowed.")
 363             sys.exit(1)
 364
 365         if apkfilename in apkcache:
 366             logging.debug("Reading " + apkfilename + " from cache")
 367             thisinfo = apkcache[apkfilename]
 368
 369         else:
 370             logging.debug("Processing " + apkfilename)
 371             thisinfo = {}
 372             thisinfo['apkname'] = apkfilename
 373             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 374             if os.path.exists(os.path.join(repodir, srcfilename)):
 375                 thisinfo['srcname'] = srcfilename
 376             thisinfo['size'] = os.path.getsize(apkfile)
 377             thisinfo['permissions'] = set()
 378             thisinfo['features'] = set()
 379             thisinfo['icons_src'] = {}
 380             thisinfo['icons'] = {}
 381             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 382             if p.returncode != 0:
 383                 if options.delete_unknown:
 384                     if os.path.exists(apkfile):
 385                         logging.error("Failed to get apk information, deleting " + apkfile)
 386                         os.remove(apkfile)
 387                     else:
 388                         logging.error("Could not find {0} to remove it".format(apkfile))
 389                 else:
 390                     logging.error("Failed to get apk information, skipping " + apkfile)
 391                 continue
 392             for line in p.output.splitlines():
 393                 if line.startswith("package:"):
 394                     try:
 395                         thisinfo['id'] = re.match(name_pat, line).group(1)
 396                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 397                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 398                     except Exception, e:
 399                         logging.error("Package matching failed: " + str(e))
 400                         logging.info("Line was: " + line)
 401                         sys.exit(1)
 402                 elif line.startswith("application:"):
 403                     thisinfo['name'] = re.match(label_pat, line).group(1)
 404                     # Keep path to non-dpi icon in case we need it
 405                     match = re.match(icon_pat_nodpi, line)
 406                     if match:
 407                         thisinfo['icons_src']['-1'] = match.group(1)
 408                 elif line.startswith("launchable-activity:"):
 409                     # Only use launchable-activity as fallback to application
 410                     if not thisinfo['name']:
 411                         thisinfo['name'] = re.match(label_pat, line).group(1)
 412                     if '-1' not in thisinfo['icons_src']:
 413                         match = re.match(icon_pat_nodpi, line)
 414                         if match:
 415                             thisinfo['icons_src']['-1'] = match.group(1)
 416                 elif line.startswith("application-icon-"):
 417                     match = re.match(icon_pat, line)
 418                     if match:
 419                         density = match.group(1)
 420                         path = match.group(2)
 421                         thisinfo['icons_src'][density] = path
 422                 elif line.startswith("sdkVersion:"):
 423                     m = re.match(sdkversion_pat, line)
 424                     if m is None:
 425                         logging.error(line.replace('sdkVersion:', '')
 426                                       + ' is not a valid minSdkVersion!')
 427                     else:
 428                         thisinfo['sdkversion'] = m.group(1)
 429                 elif line.startswith("maxSdkVersion:"):
 430                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 431                 elif line.startswith("native-code:"):
 432                     thisinfo['nativecode'] = []
 433                     for arch in line[13:].split(' '):
 434                         thisinfo['nativecode'].append(arch[1:-1])
 435                 elif line.startswith("uses-permission:"):
 436                     perm = re.match(string_pat, line).group(1)
 437                     if perm.startswith("android.permission."):
 438                         perm = perm[19:]
 439                     thisinfo['permissions'].add(perm)
 440                 elif line.startswith("uses-feature:"):
 441                     perm = re.match(string_pat, line).group(1)
 442                     # Filter out this, it's only added with the latest SDK tools and
 443                     # causes problems for lots of apps.
 444                     if perm != "android.hardware.screen.portrait" \
 445                             and perm != "android.hardware.screen.landscape":
 446                         if perm.startswith("android.feature."):
 447                             perm = perm[16:]
 448                         thisinfo['features'].add(perm)
 449
 450             if 'sdkversion' not in thisinfo:
 451                 logging.warn("No SDK version information found in {0}".format(apkfile))
 452                 thisinfo['sdkversion'] = 0
 453
 454             # Check for debuggable apks...
 455             if common.isApkDebuggable(apkfile, config):
 456                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 457
 458             # Calculate the sha256...
 459             sha = hashlib.sha256()
 460             with open(apkfile, 'rb') as f:
 461                 while True:
 462                     t = f.read(1024)
 463                     if len(t) == 0:
 464                         break
 465                     sha.update(t)
 466                 thisinfo['sha256'] = sha.hexdigest()
 467
 468             # Get the signature (or md5 of, to be precise)...
 469             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 470             if not os.path.exists(getsig_dir + "/getsig.class"):
 471                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 472                 sys.exit(1)
 473             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 474                              'getsig', os.path.join(os.getcwd(), apkfile)])
 475             thisinfo['sig'] = None
 476             for line in p.output.splitlines():
 477                 if line.startswith('Result:'):
 478                     thisinfo['sig'] = line[7:].strip()
 479                     break
 480             if p.returncode != 0 or not thisinfo['sig']:
 481                 logging.critical("Failed to get apk signature")
 482                 sys.exit(1)
 483
 484             apk = zipfile.ZipFile(apkfile, 'r')
 485
 486             iconfilename = "%s.%s.png" % (
 487                 thisinfo['id'],
 488                 thisinfo['versioncode'])
 489
 490             # Extract the icon file...
 491             densities = get_densities()
 492             empty_densities = []
 493             for density in densities:
 494                 if density not in thisinfo['icons_src']:
 495                     empty_densities.append(density)
 496                     continue
 497                 iconsrc = thisinfo['icons_src'][density]
 498                 icon_dir = get_icon_dir(repodir, density)
 499                 icondest = os.path.join(icon_dir, iconfilename)
 500
 501                 try:
 502                     iconfile = open(icondest, 'wb')
 503                     iconfile.write(apk.read(iconsrc))
 504                     iconfile.close()
 505                     thisinfo['icons'][density] = iconfilename
 506
 507                 except:
 508                     logging.warn("Error retrieving icon file")
 509                     del thisinfo['icons'][density]
 510                     del thisinfo['icons_src'][density]
 511                     empty_densities.append(density)
 512
 513             if '-1' in thisinfo['icons_src']:
 514                 iconsrc = thisinfo['icons_src']['-1']
 515                 iconpath = os.path.join(
 516                     get_icon_dir(repodir, None), iconfilename)
 517                 iconfile = open(iconpath, 'wb')
 518                 iconfile.write(apk.read(iconsrc))
 519                 iconfile.close()
 520                 try:
 521                     im = Image.open(iconpath)
 522                     dpi = px_to_dpi(im.size[0])
 523                     for density in densities:
 524                         if density in thisinfo['icons']:
 525                             break
 526                         if density == densities[-1] or dpi >= int(density):
 527                             thisinfo['icons'][density] = iconfilename
 528                             shutil.move(iconpath,
 529                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 530                             empty_densities.remove(density)
 531                             break
 532                 except Exception, e:
 533                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 534
 535             if thisinfo['icons']:
 536                 thisinfo['icon'] = iconfilename
 537
 538             apk.close()
 539
 540             # First try resizing down to not lose quality
 541             last_density = None
 542             for density in densities:
 543                 if density not in empty_densities:
 544                     last_density = density
 545                     continue
 546                 if last_density is None:
 547                     continue
 548                 logging.debug("Density %s not available, resizing down from %s"
 549                               % (density, last_density))
 550
 551                 last_iconpath = os.path.join(
 552                     get_icon_dir(repodir, last_density), iconfilename)
 553                 iconpath = os.path.join(
 554                     get_icon_dir(repodir, density), iconfilename)
 555                 try:
 556                     im = Image.open(last_iconpath)
 557                 except:
 558                     logging.warn("Invalid image file at %s" % last_iconpath)
 559                     continue
 560
 561                 size = dpi_to_px(density)
 562
 563                 im.thumbnail((size, size), Image.ANTIALIAS)
 564                 im.save(iconpath, "PNG")
 565                 empty_densities.remove(density)
 566
 567             # Then just copy from the highest resolution available
 568             last_density = None
 569             for density in reversed(densities):
 570                 if density not in empty_densities:
 571                     last_density = density
 572                     continue
 573                 if last_density is None:
 574                     continue
 575                 logging.debug("Density %s not available, copying from lower density %s"
 576                               % (density, last_density))
 577
 578                 shutil.copyfile(
 579                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 580                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 581
 582                 empty_densities.remove(density)
 583
 584             for density in densities:
 585                 icon_dir = get_icon_dir(repodir, density)
 586                 icondest = os.path.join(icon_dir, iconfilename)
 587                 resize_icon(icondest, density)
 588
 589             # Copy from icons-mdpi to icons since mdpi is the baseline density
 590             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 591             if os.path.isfile(baseline):
 592                 shutil.copyfile(baseline,
 593                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 594
 595             # Record in known apks, getting the added date at the same time..
 596             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 597             if added:
 598                 thisinfo['added'] = added
 599
 600             apkcache[apkfilename] = thisinfo
 601             cachechanged = True
 602
 603         apks.append(thisinfo)
 604
 605     return apks, cachechanged
 606
 607
 608 repo_pubkey_fingerprint = None
 609
 610
 611 def make_index(apps, sortedids, apks, repodir, archive, categories):
 612     """Make a repo index.
 613
 614     :param apps: fully populated apps list
 615     :param apks: full populated apks list
 616     :param repodir: the repo directory
 617     :param archive: True if this is the archive repo, False if it's the
 618                     main one.
 619     :param categories: list of categories
 620     """
 621
 622     doc = Document()
 623
 624     def addElement(name, value, doc, parent):
 625         el = doc.createElement(name)
 626         el.appendChild(doc.createTextNode(value))
 627         parent.appendChild(el)
 628
 629     def addElementCDATA(name, value, doc, parent):
 630         el = doc.createElement(name)
 631         el.appendChild(doc.createCDATASection(value))
 632         parent.appendChild(el)
 633
 634     root = doc.createElement("fdroid")
 635     doc.appendChild(root)
 636
 637     repoel = doc.createElement("repo")
 638
 639     if archive:
 640         repoel.setAttribute("name", config['archive_name'])
 641         if config['repo_maxage'] != 0:
 642             repoel.setAttribute("maxage", str(config['repo_maxage']))
 643         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 644         repoel.setAttribute("url", config['archive_url'])
 645         addElement('description', config['archive_description'], doc, repoel)
 646
 647     else:
 648         repoel.setAttribute("name", config['repo_name'])
 649         if config['repo_maxage'] != 0:
 650             repoel.setAttribute("maxage", str(config['repo_maxage']))
 651         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 652         repoel.setAttribute("url", config['repo_url'])
 653         addElement('description', config['repo_description'], doc, repoel)
 654
 655     repoel.setAttribute("version", "12")
 656     repoel.setAttribute("timestamp", str(int(time.time())))
 657
 658     if 'repo_keyalias' in config:
 659
 660         # Generate a certificate fingerprint the same way keytool does it
 661         # (but with slightly different formatting)
 662         def cert_fingerprint(data):
 663             digest = hashlib.sha256(data).digest()
 664             ret = []
 665             ret.append(' '.join("%02X" % ord(b) for b in digest))
 666             return " ".join(ret)
 667
 668         def extract_pubkey():
 669             p = FDroidPopen(['keytool', '-exportcert',
 670                              '-alias', config['repo_keyalias'],
 671                              '-keystore', config['keystore'],
 672                              '-storepass:file', config['keystorepassfile']]
 673                             + config['smartcardoptions'], output=False)
 674             if p.returncode != 0:
 675                 msg = "Failed to get repo pubkey!"
 676                 if config['keystore'] == 'NONE':
 677                     msg += ' Is your crypto smartcard plugged in?'
 678                 logging.critical(msg)
 679                 sys.exit(1)
 680             global repo_pubkey_fingerprint
 681             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 682             return "".join("%02x" % ord(b) for b in p.output)
 683
 684         repoel.setAttribute("pubkey", extract_pubkey())
 685
 686     root.appendChild(repoel)
 687
 688     for appid in sortedids:
 689         app = apps[appid]
 690
 691         if app['Disabled'] is not None:
 692             continue
 693
 694         # Get a list of the apks for this app...
 695         apklist = []
 696         for apk in apks:
 697             if apk['id'] == appid:
 698                 apklist.append(apk)
 699
 700         if len(apklist) == 0:
 701             continue
 702
 703         apel = doc.createElement("application")
 704         apel.setAttribute("id", app['id'])
 705         root.appendChild(apel)
 706
 707         addElement('id', app['id'], doc, apel)
 708         if 'added' in app:
 709             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 710         if 'lastupdated' in app:
 711             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 712         addElement('name', app['Name'], doc, apel)
 713         addElement('summary', app['Summary'], doc, apel)
 714         if app['icon']:
 715             addElement('icon', app['icon'], doc, apel)
 716
 717         def linkres(appid):
 718             if appid in apps:
 719                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 720             raise MetaDataException("Cannot resolve app id " + appid)
 721
 722         addElement('desc',
 723                    metadata.description_html(app['Description'], linkres),
 724                    doc, apel)
 725         addElement('license', app['License'], doc, apel)
 726         if 'Categories' in app:
 727             addElement('categories', ','.join(app["Categories"]), doc, apel)
 728             # We put the first (primary) category in LAST, which will have
 729             # the desired effect of making clients that only understand one
 730             # category see that one.
 731             addElement('category', app["Categories"][0], doc, apel)
 732         addElement('web', app['Web Site'], doc, apel)
 733         addElement('source', app['Source Code'], doc, apel)
 734         addElement('tracker', app['Issue Tracker'], doc, apel)
 735         if app['Donate']:
 736             addElement('donate', app['Donate'], doc, apel)
 737         if app['Bitcoin']:
 738             addElement('bitcoin', app['Bitcoin'], doc, apel)
 739         if app['Litecoin']:
 740             addElement('litecoin', app['Litecoin'], doc, apel)
 741         if app['Dogecoin']:
 742             addElement('dogecoin', app['Dogecoin'], doc, apel)
 743         if app['FlattrID']:
 744             addElement('flattr', app['FlattrID'], doc, apel)
 745
 746         # These elements actually refer to the current version (i.e. which
 747         # one is recommended. They are historically mis-named, and need
 748         # changing, but stay like this for now to support existing clients.
 749         addElement('marketversion', app['Current Version'], doc, apel)
 750         addElement('marketvercode', app['Current Version Code'], doc, apel)
 751
 752         if app['AntiFeatures']:
 753             af = app['AntiFeatures'].split(',')
 754             # TODO: Temporarily not including UpstreamNonFree in the index,
 755             # because current F-Droid clients do not understand it, and also
 756             # look ugly when they encounter an unknown antifeature. This
 757             # filtering can be removed in time...
 758             if 'UpstreamNonFree' in af:
 759                 af.remove('UpstreamNonFree')
 760             if af:
 761                 addElement('antifeatures', ','.join(af), doc, apel)
 762         if app['Provides']:
 763             pv = app['Provides'].split(',')
 764             addElement('provides', ','.join(pv), doc, apel)
 765         if app['Requires Root']:
 766             addElement('requirements', 'root', doc, apel)
 767
 768         # Sort the apk list into version order, just so the web site
 769         # doesn't have to do any work by default...
 770         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 771
 772         # Check for duplicates - they will make the client unhappy...
 773         for i in range(len(apklist) - 1):
 774             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 775                 logging.critical("duplicate versions: '%s' - '%s'" % (
 776                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 777                 sys.exit(1)
 778
 779         for apk in apklist:
 780             apkel = doc.createElement("package")
 781             apel.appendChild(apkel)
 782             addElement('version', apk['version'], doc, apkel)
 783             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 784             addElement('apkname', apk['apkname'], doc, apkel)
 785             if 'srcname' in apk:
 786                 addElement('srcname', apk['srcname'], doc, apkel)
 787             for hash_type in ['sha256']:
 788                 if hash_type not in apk:
 789                     continue
 790                 hashel = doc.createElement("hash")
 791                 hashel.setAttribute("type", hash_type)
 792                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 793                 apkel.appendChild(hashel)
 794             addElement('sig', apk['sig'], doc, apkel)
 795             addElement('size', str(apk['size']), doc, apkel)
 796             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 797             if 'maxsdkversion' in apk:
 798                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 799             if 'added' in apk:
 800                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 801             if app['Requires Root']:
 802                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 803                     apk['permissions'].add('ACCESS_SUPERUSER')
 804
 805             if len(apk['permissions']) > 0:
 806                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 807             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 808                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 809             if len(apk['features']) > 0:
 810                 addElement('features', ','.join(apk['features']), doc, apkel)
 811
 812     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 813     if options.pretty:
 814         output = doc.toprettyxml()
 815     else:
 816         output = doc.toxml()
 817     of.write(output)
 818     of.close()
 819
 820     if 'repo_keyalias' in config:
 821
 822         logging.info("Creating signed index with this key (SHA256):")
 823         logging.info("%s" % repo_pubkey_fingerprint)
 824
 825         # Create a jar of the index...
 826         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 827         if p.returncode != 0:
 828             logging.critical("Failed to create jar file")
 829             sys.exit(1)
 830
 831         # Sign the index...
 832         args = ['jarsigner', '-keystore', config['keystore'],
 833                 '-storepass:file', config['keystorepassfile'],
 834                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 835                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 836         if config['keystore'] == 'NONE':
 837             args += config['smartcardoptions']
 838         else:  # smardcards never use -keypass
 839             args += ['-keypass:file', config['keypassfile']]
 840         p = FDroidPopen(args)
 841         # TODO keypass should be sent via stdin
 842         if p.returncode != 0:
 843             logging.critical("Failed to sign index")
 844             sys.exit(1)
 845
 846     # Copy the repo icon into the repo directory...
 847     icon_dir = os.path.join(repodir, 'icons')
 848     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 849     shutil.copyfile(config['repo_icon'], iconfilename)
 850
 851     # Write a category list in the repo to allow quick access...
 852     catdata = ''
 853     for cat in categories:
 854         catdata += cat + '\n'
 855     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 856     f.write(catdata)
 857     f.close()
 858
 859
 860 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 861
 862     for appid, app in apps.iteritems():
 863
 864         # Get a list of the apks for this app...
 865         apklist = []
 866         for apk in apks:
 867             if apk['id'] == appid:
 868                 apklist.append(apk)
 869
 870         # Sort the apk list into version order...
 871         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 872
 873         if app['Archive Policy']:
 874             keepversions = int(app['Archive Policy'][:-9])
 875         else:
 876             keepversions = defaultkeepversions
 877
 878         if len(apklist) > keepversions:
 879             for apk in apklist[keepversions:]:
 880                 logging.info("Moving " + apk['apkname'] + " to archive")
 881                 shutil.move(os.path.join(repodir, apk['apkname']),
 882                             os.path.join(archivedir, apk['apkname']))
 883                 if 'srcname' in apk:
 884                     shutil.move(os.path.join(repodir, apk['srcname']),
 885                                 os.path.join(archivedir, apk['srcname']))
 886                     # Move GPG signature too...
 887                     sigfile = apk['srcname'] + '.asc'
 888                     sigsrc = os.path.join(repodir, sigfile)
 889                     if os.path.exists(sigsrc):
 890                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 891
 892                 archapks.append(apk)
 893                 apks.remove(apk)
 894
 895
 896 config = None
 897 options = None
 898
 899
 900 def main():
 901
 902     global config, options
 903
 904     # Parse command line...
 905     parser = OptionParser()
 906     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 907                       help="Create skeleton metadata files that are missing")
 908     parser.add_option("--delete-unknown", action="store_true", default=False,
 909                       help="Delete APKs without metadata from the repo")
 910     parser.add_option("-v", "--verbose", action="store_true", default=False,
 911                       help="Spew out even more information than normal")
 912     parser.add_option("-q", "--quiet", action="store_true", default=False,
 913                       help="Restrict output to warnings and errors")
 914     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 915                       help="Report on build data status")
 916     parser.add_option("-i", "--interactive", default=False, action="store_true",
 917                       help="Interactively ask about things that need updating.")
 918     parser.add_option("-I", "--icons", action="store_true", default=False,
 919                       help="Resize all the icons exceeding the max pixel size and exit")
 920     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 921                       help="Specify editor to use in interactive mode. Default " +
 922                       "is /etc/alternatives/editor")
 923     parser.add_option("-w", "--wiki", default=False, action="store_true",
 924                       help="Update the wiki")
 925     parser.add_option("", "--pretty", action="store_true", default=False,
 926                       help="Produce human-readable index.xml")
 927     parser.add_option("--clean", action="store_true", default=False,
 928                       help="Clean update - don't uses caches, reprocess all apks")
 929     (options, args) = parser.parse_args()
 930
 931     config = common.read_config(options)
 932
 933     repodirs = ['repo']
 934     if config['archive_older'] != 0:
 935         repodirs.append('archive')
 936         if not os.path.exists('archive'):
 937             os.mkdir('archive')
 938
 939     if options.icons:
 940         resize_all_icons(repodirs)
 941         sys.exit(0)
 942
 943     # check that icons exist now, rather than fail at the end of `fdroid update`
 944     for k in ['repo_icon', 'archive_icon']:
 945         if k in config:
 946             if not os.path.exists(config[k]):
 947                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 948                 sys.exit(1)
 949
 950     # Get all apps...
 951     apps = metadata.read_metadata()
 952
 953     # Generate a list of categories...
 954     categories = set()
 955     for app in apps.itervalues():
 956         categories.update(app['Categories'])
 957
 958     # Read known apks data (will be updated and written back when we've finished)
 959     knownapks = common.KnownApks()
 960
 961     # Gather information about all the apk files in the repo directory, using
 962     # cached data if possible.
 963     apkcachefile = os.path.join('tmp', 'apkcache')
 964     if not options.clean and os.path.exists(apkcachefile):
 965         with open(apkcachefile, 'rb') as cf:
 966             apkcache = pickle.load(cf)
 967     else:
 968         apkcache = {}
 969     cachechanged = False
 970
 971     delete_disabled_builds(apps, apkcache, repodirs)
 972
 973     # Scan all apks in the main repo
 974     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 975     if cc:
 976         cachechanged = True
 977
 978     # Generate warnings for apk's with no metadata (or create skeleton
 979     # metadata files, if requested on the command line)
 980     newmetadata = False
 981     for apk in apks:
 982         if apk['id'] not in apps:
 983             if options.create_metadata:
 984                 if 'name' not in apk:
 985                     logging.error(apk['id'] + ' does not have a name! Skipping...')
 986                     continue
 987                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 988                 f.write("License:Unknown\n")
 989                 f.write("Web Site:\n")
 990                 f.write("Source Code:\n")
 991                 f.write("Issue Tracker:\n")
 992                 f.write("Summary:" + apk['name'] + "\n")
 993                 f.write("Description:\n")
 994                 f.write(apk['name'] + "\n")
 995                 f.write(".\n")
 996                 f.close()
 997                 logging.info("Generated skeleton metadata for " + apk['id'])
 998                 newmetadata = True
 999             else:
1000                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1001                 if options.delete_unknown:
1002                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1003                     rmf = os.path.join(repodirs[0], apk['apkname'])
1004                     if not os.path.exists(rmf):
1005                         logging.error("Could not find {0} to remove it".format(rmf))
1006                     else:
1007                         os.remove(rmf)
1008                 else:
1009                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1010
1011     # update the metadata with the newly created ones included
1012     if newmetadata:
1013         apps = metadata.read_metadata()
1014
1015     # Scan the archive repo for apks as well
1016     if len(repodirs) > 1:
1017         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1018         if cc:
1019             cachechanged = True
1020     else:
1021         archapks = []
1022
1023     # Some information from the apks needs to be applied up to the application
1024     # level. When doing this, we use the info from the most recent version's apk.
1025     # We deal with figuring out when the app was added and last updated at the
1026     # same time.
1027     for appid, app in apps.iteritems():
1028         bestver = 0
1029         added = None
1030         lastupdated = None
1031         for apk in apks + archapks:
1032             if apk['id'] == appid:
1033                 if apk['versioncode'] > bestver:
1034                     bestver = apk['versioncode']
1035                     bestapk = apk
1036
1037                 if 'added' in apk:
1038                     if not added or apk['added'] < added:
1039                         added = apk['added']
1040                     if not lastupdated or apk['added'] > lastupdated:
1041                         lastupdated = apk['added']
1042
1043         if added:
1044             app['added'] = added
1045         else:
1046             logging.warn("Don't know when " + appid + " was added")
1047         if lastupdated:
1048             app['lastupdated'] = lastupdated
1049         else:
1050             logging.warn("Don't know when " + appid + " was last updated")
1051
1052         if bestver == 0:
1053             if app['Name'] is None:
1054                 app['Name'] = app['Auto Name'] or appid
1055             app['icon'] = None
1056             logging.warn("Application " + appid + " has no packages")
1057         else:
1058             if app['Name'] is None:
1059                 app['Name'] = bestapk['name']
1060             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1061
1062     # Sort the app list by name, then the web site doesn't have to by default.
1063     # (we had to wait until we'd scanned the apks to do this, because mostly the
1064     # name comes from there!)
1065     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1066
1067     if len(repodirs) > 1:
1068         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1069
1070     # Make the index for the main repo...
1071     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1072
1073     # If there's an archive repo,  make the index for it. We already scanned it
1074     # earlier on.
1075     if len(repodirs) > 1:
1076         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1077
1078     if config['update_stats']:
1079
1080         # Update known apks info...
1081         knownapks.writeifchanged()
1082
1083         # Generate latest apps data for widget
1084         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1085             data = ''
1086             for line in file(os.path.join('stats', 'latestapps.txt')):
1087                 appid = line.rstrip()
1088                 data += appid + "\t"
1089                 app = apps[appid]
1090                 data += app['Name'] + "\t"
1091                 if app['icon'] is not None:
1092                     data += app['icon'] + "\t"
1093                 data += app['License'] + "\n"
1094             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1095             f.write(data)
1096             f.close()
1097
1098     if cachechanged:
1099         with open(apkcachefile, 'wb') as cf:
1100             pickle.dump(apkcache, cf)
1101
1102     # Update the wiki...
1103     if options.wiki:
1104         update_wiki(apps, sortedids, apks + archapks)
1105
1106     logging.info("Finished.")
1107
1108 if __name__ == "__main__":
1109     main()