fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48 screen_densities = ['640', '480', '320', '240', '160', '120']
  49
  50 all_screen_densities = ['0'] + screen_densities
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density == '0':
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in screen_densities:
  69         yield get_icon_dir(repodir, density)
  70
  71
  72 def get_all_icon_dirs(repodir):
  73     for density in all_screen_densities:
  74         yield get_icon_dir(repodir, density)
  75
  76
  77 def update_wiki(apps, sortedids, apks):
  78     """Update the wiki
  79
  80     :param apps: fully populated list of all applications
  81     :param apks: all apks, except...
  82     """
  83     logging.info("Updating wiki")
  84     wikicat = 'Apps'
  85     wikiredircat = 'App Redirects'
  86     import mwclient
  87     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  88                          path=config['wiki_path'])
  89     site.login(config['wiki_user'], config['wiki_password'])
  90     generated_pages = {}
  91     generated_redirects = {}
  92
  93     for appid in sortedids:
  94         app = apps[appid]
  95
  96         wikidata = ''
  97         if app['Disabled']:
  98             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  99         if 'AntiFeatures' in app:
 100             for af in app['AntiFeatures']:
 101                 wikidata += '{{AntiFeature|' + af + '}}\n'
 102         if app['Requires Root']:
 103             requiresroot = 'Yes'
 104         else:
 105             requiresroot = 'No'
 106         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
 107             appid,
 108             app['Name'],
 109             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 110             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 111             app['Source Code'],
 112             app['Issue Tracker'],
 113             app['Web Site'],
 114             app['Changelog'],
 115             app['Donate'],
 116             app['FlattrID'],
 117             app['Bitcoin'],
 118             app['Litecoin'],
 119             app['License'],
 120             requiresroot)
 121
 122         if app['Provides']:
 123             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 124
 125         wikidata += app['Summary']
 126         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 127
 128         wikidata += "=Description=\n"
 129         wikidata += metadata.description_wiki(app['Description']) + "\n"
 130
 131         wikidata += "=Maintainer Notes=\n"
 132         if 'Maintainer Notes' in app:
 133             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 134         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 135
 136         # Get a list of all packages for this application...
 137         apklist = []
 138         gotcurrentver = False
 139         cantupdate = False
 140         buildfails = False
 141         for apk in apks:
 142             if apk['id'] == appid:
 143                 if str(apk['versioncode']) == app['Current Version Code']:
 144                     gotcurrentver = True
 145                 apklist.append(apk)
 146         # Include ones we can't build, as a special case...
 147         for thisbuild in app['builds']:
 148             if thisbuild['disable']:
 149                 if thisbuild['vercode'] == app['Current Version Code']:
 150                     cantupdate = True
 151                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 152                 apklist.append({'versioncode': int(thisbuild['vercode']),
 153                                 'version': thisbuild['version'],
 154                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
 155                                 })
 156             else:
 157                 builtit = False
 158                 for apk in apklist:
 159                     if apk['versioncode'] == int(thisbuild['vercode']):
 160                         builtit = True
 161                         break
 162                 if not builtit:
 163                     buildfails = True
 164                     apklist.append({'versioncode': int(thisbuild['vercode']),
 165                                     'version': thisbuild['version'],
 166                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
 167                                     })
 168         if app['Current Version Code'] == '0':
 169             cantupdate = True
 170         # Sort with most recent first...
 171         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 172
 173         wikidata += "=Versions=\n"
 174         if len(apklist) == 0:
 175             wikidata += "We currently have no versions of this app available."
 176         elif not gotcurrentver:
 177             wikidata += "We don't have the current version of this app."
 178         else:
 179             wikidata += "We have the current version of this app."
 180         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 181         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 182         if len(app['No Source Since']) > 0:
 183             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 184         if len(app['Current Version']) > 0:
 185             wikidata += "The current (recommended) version is " + app['Current Version']
 186             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 187         validapks = 0
 188         for apk in apklist:
 189             wikidata += "==" + apk['version'] + "==\n"
 190
 191             if 'buildproblem' in apk:
 192                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 193             else:
 194                 validapks += 1
 195                 wikidata += "This version is built and signed by "
 196                 if 'srcname' in apk:
 197                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 198                 else:
 199                     wikidata += "the original developer.\n\n"
 200             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 201
 202         wikidata += '\n[[Category:' + wikicat + ']]\n'
 203         if len(app['No Source Since']) > 0:
 204             wikidata += '\n[[Category:Apps missing source code]]\n'
 205         if validapks == 0 and not app['Disabled']:
 206             wikidata += '\n[[Category:Apps with no packages]]\n'
 207         if cantupdate and not app['Disabled']:
 208             wikidata += "\n[[Category:Apps we can't update]]\n"
 209         if buildfails and not app['Disabled']:
 210             wikidata += "\n[[Category:Apps with failing builds]]\n"
 211         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 212             wikidata += '\n[[Category:Apps to Update]]\n'
 213         if app['Disabled']:
 214             wikidata += '\n[[Category:Apps that are disabled]]\n'
 215         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 216             wikidata += '\n[[Category:Apps with no update check]]\n'
 217         for appcat in app['Categories']:
 218             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 219
 220         # We can't have underscores in the page name, even if they're in
 221         # the package ID, because MediaWiki messes with them...
 222         pagename = appid.replace('_', ' ')
 223
 224         # Drop a trailing newline, because mediawiki is going to drop it anyway
 225         # and it we don't we'll think the page has changed when it hasn't...
 226         if wikidata.endswith('\n'):
 227             wikidata = wikidata[:-1]
 228
 229         generated_pages[pagename] = wikidata
 230
 231         # Make a redirect from the name to the ID too, unless there's
 232         # already an existing page with the name and it isn't a redirect.
 233         noclobber = False
 234         apppagename = app['Name'].replace('_', ' ')
 235         apppagename = apppagename.replace('{', '')
 236         apppagename = apppagename.replace('}', ' ')
 237         apppagename = apppagename.replace(':', ' ')
 238         # Drop double spaces caused mostly by replacing ':' above
 239         apppagename = apppagename.replace('  ', ' ')
 240         for expagename in site.allpages(prefix=apppagename,
 241                                         filterredir='nonredirects',
 242                                         generator=False):
 243             if expagename == apppagename:
 244                 noclobber = True
 245         # Another reason not to make the redirect page is if the app name
 246         # is the same as it's ID, because that will overwrite the real page
 247         # with an redirect to itself! (Although it seems like an odd
 248         # scenario this happens a lot, e.g. where there is metadata but no
 249         # builds or binaries to extract a name from.
 250         if apppagename == pagename:
 251             noclobber = True
 252         if not noclobber:
 253             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 254
 255     for tcat, genp in [(wikicat, generated_pages),
 256                        (wikiredircat, generated_redirects)]:
 257         catpages = site.Pages['Category:' + tcat]
 258         existingpages = []
 259         for page in catpages:
 260             existingpages.append(page.name)
 261             if page.name in genp:
 262                 pagetxt = page.edit()
 263                 if pagetxt != genp[page.name]:
 264                     logging.debug("Updating modified page " + page.name)
 265                     page.save(genp[page.name], summary='Auto-updated')
 266                 else:
 267                     logging.debug("Page " + page.name + " is unchanged")
 268             else:
 269                 logging.warn("Deleting page " + page.name)
 270                 page.delete('No longer published')
 271         for pagename, text in genp.items():
 272             logging.debug("Checking " + pagename)
 273             if pagename not in existingpages:
 274                 logging.debug("Creating page " + pagename)
 275                 try:
 276                     newpage = site.Pages[pagename]
 277                     newpage.save(text, summary='Auto-created')
 278                 except:
 279                     logging.error("...FAILED to create page '{0}'".format(pagename))
 280
 281     # Purge server cache to ensure counts are up to date
 282     site.pages['Repository Maintenance'].purge()
 283
 284
 285 def delete_disabled_builds(apps, apkcache, repodirs):
 286     """Delete disabled build outputs.
 287
 288     :param apps: list of all applications, as per metadata.read_metadata
 289     :param apkcache: current apk cache information
 290     :param repodirs: the repo directories to process
 291     """
 292     for appid, app in apps.iteritems():
 293         for build in app['builds']:
 294             if build['disable']:
 295                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 296                 for repodir in repodirs:
 297                     apkpath = os.path.join(repodir, apkfilename)
 298                     ascpath = apkpath + ".asc"
 299                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 300                     for name in [apkpath, srcpath, ascpath]:
 301                         if os.path.exists(name):
 302                             logging.warn("Deleting disabled build output " + apkfilename)
 303                             os.remove(name)
 304                 if apkfilename in apkcache:
 305                     del apkcache[apkfilename]
 306
 307
 308 def resize_icon(iconpath, density):
 309
 310     if not os.path.isfile(iconpath):
 311         return
 312
 313     try:
 314         im = Image.open(iconpath)
 315         size = dpi_to_px(density)
 316
 317         if any(length > size for length in im.size):
 318             oldsize = im.size
 319             im.thumbnail((size, size), Image.ANTIALIAS)
 320             logging.debug("%s was too large at %s - new size is %s" % (
 321                 iconpath, oldsize, im.size))
 322             im.save(iconpath, "PNG")
 323
 324     except Exception, e:
 325         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 326
 327
 328 def resize_all_icons(repodirs):
 329     """Resize all icons that exceed the max size
 330
 331     :param repodirs: the repo directories to process
 332     """
 333     for repodir in repodirs:
 334         for density in screen_densities:
 335             icon_dir = get_icon_dir(repodir, density)
 336             icon_glob = os.path.join(icon_dir, '*.png')
 337             for iconpath in glob.glob(icon_glob):
 338                 resize_icon(iconpath, density)
 339
 340
 341 # A signature block file with a .DSA, .RSA, or .EC extension
 342 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 343
 344
 345 def getsig(apkpath):
 346     """ Get the signing certificate of an apk. To get the same md5 has that
 347     Android gets, we encode the .RSA certificate in a specific format and pass
 348     it hex-encoded to the md5 digest algorithm.
 349
 350     :param apkpath: path to the apk
 351     :returns: A string containing the md5 of the signature of the apk or None
 352               if an error occurred.
 353     """
 354
 355     cert = None
 356
 357     # verify the jar signature is correct
 358     args = ['jarsigner', '-verify', apkpath]
 359     p = FDroidPopen(args)
 360     if p.returncode != 0:
 361         logging.critical(apkpath + " has a bad signature!")
 362         return None
 363
 364     with zipfile.ZipFile(apkpath, 'r') as apk:
 365
 366         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 367
 368         if len(certs) < 1:
 369             logging.error("Found no signing certificates on %s" % apkpath)
 370             return None
 371         if len(certs) > 1:
 372             logging.error("Found multiple signing certificates on %s" % apkpath)
 373             return None
 374
 375         cert = apk.read(certs[0])
 376
 377     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 378     if content.getComponentByName('contentType') != rfc2315.signedData:
 379         logging.error("Unexpected format.")
 380         return None
 381
 382     content = decoder.decode(content.getComponentByName('content'),
 383                              asn1Spec=rfc2315.SignedData())[0]
 384     try:
 385         certificates = content.getComponentByName('certificates')
 386     except PyAsn1Error:
 387         logging.error("Certificates not found.")
 388         return None
 389
 390     cert_encoded = encoder.encode(certificates)[4:]
 391
 392     return md5(cert_encoded.encode('hex')).hexdigest()
 393
 394
 395 def scan_apks(apps, apkcache, repodir, knownapks):
 396     """Scan the apks in the given repo directory.
 397
 398     This also extracts the icons.
 399
 400     :param apps: list of all applications, as per metadata.read_metadata
 401     :param apkcache: current apk cache information
 402     :param repodir: repo directory to scan
 403     :param knownapks: known apks info
 404     :returns: (apks, cachechanged) where apks is a list of apk information,
 405               and cachechanged is True if the apkcache got changed.
 406     """
 407
 408     cachechanged = False
 409
 410     for icon_dir in get_all_icon_dirs(repodir):
 411         if os.path.exists(icon_dir):
 412             if options.clean:
 413                 shutil.rmtree(icon_dir)
 414                 os.makedirs(icon_dir)
 415         else:
 416             os.makedirs(icon_dir)
 417
 418     apks = []
 419     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 420     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 421     vername_pat = re.compile(".*versionName='([^']*)'.*")
 422     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 423     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 424     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 425     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 426     string_pat = re.compile(".*'([^']*)'.*")
 427     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 428
 429         apkfilename = apkfile[len(repodir) + 1:]
 430         if ' ' in apkfilename:
 431             logging.critical("Spaces in filenames are not allowed.")
 432             sys.exit(1)
 433
 434         # Calculate the sha256...
 435         sha = hashlib.sha256()
 436         with open(apkfile, 'rb') as f:
 437             while True:
 438                 t = f.read(16384)
 439                 if len(t) == 0:
 440                     break
 441                 sha.update(t)
 442             shasum = sha.hexdigest()
 443
 444         usecache = False
 445         if apkfilename in apkcache:
 446             thisinfo = apkcache[apkfilename]
 447             if thisinfo['sha256'] == shasum:
 448                 logging.debug("Reading " + apkfilename + " from cache")
 449                 usecache = True
 450             else:
 451                 logging.debug("Ignoring stale cache data for " + apkfilename)
 452
 453         if not usecache:
 454             logging.debug("Processing " + apkfilename)
 455             thisinfo = {}
 456             thisinfo['apkname'] = apkfilename
 457             thisinfo['sha256'] = shasum
 458             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 459             if os.path.exists(os.path.join(repodir, srcfilename)):
 460                 thisinfo['srcname'] = srcfilename
 461             thisinfo['size'] = os.path.getsize(apkfile)
 462             thisinfo['permissions'] = set()
 463             thisinfo['features'] = set()
 464             thisinfo['icons_src'] = {}
 465             thisinfo['icons'] = {}
 466             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 467             if p.returncode != 0:
 468                 if options.delete_unknown:
 469                     if os.path.exists(apkfile):
 470                         logging.error("Failed to get apk information, deleting " + apkfile)
 471                         os.remove(apkfile)
 472                     else:
 473                         logging.error("Could not find {0} to remove it".format(apkfile))
 474                 else:
 475                     logging.error("Failed to get apk information, skipping " + apkfile)
 476                 continue
 477             for line in p.output.splitlines():
 478                 if line.startswith("package:"):
 479                     try:
 480                         thisinfo['id'] = re.match(name_pat, line).group(1)
 481                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 482                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 483                     except Exception, e:
 484                         logging.error("Package matching failed: " + str(e))
 485                         logging.info("Line was: " + line)
 486                         sys.exit(1)
 487                 elif line.startswith("application:"):
 488                     thisinfo['name'] = re.match(label_pat, line).group(1)
 489                     # Keep path to non-dpi icon in case we need it
 490                     match = re.match(icon_pat_nodpi, line)
 491                     if match:
 492                         thisinfo['icons_src']['-1'] = match.group(1)
 493                 elif line.startswith("launchable-activity:"):
 494                     # Only use launchable-activity as fallback to application
 495                     if not thisinfo['name']:
 496                         thisinfo['name'] = re.match(label_pat, line).group(1)
 497                     if '-1' not in thisinfo['icons_src']:
 498                         match = re.match(icon_pat_nodpi, line)
 499                         if match:
 500                             thisinfo['icons_src']['-1'] = match.group(1)
 501                 elif line.startswith("application-icon-"):
 502                     match = re.match(icon_pat, line)
 503                     if match:
 504                         density = match.group(1)
 505                         path = match.group(2)
 506                         thisinfo['icons_src'][density] = path
 507                 elif line.startswith("sdkVersion:"):
 508                     m = re.match(sdkversion_pat, line)
 509                     if m is None:
 510                         logging.error(line.replace('sdkVersion:', '')
 511                                       + ' is not a valid minSdkVersion!')
 512                     else:
 513                         thisinfo['sdkversion'] = m.group(1)
 514                 elif line.startswith("maxSdkVersion:"):
 515                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 516                 elif line.startswith("native-code:"):
 517                     thisinfo['nativecode'] = []
 518                     for arch in line[13:].split(' '):
 519                         thisinfo['nativecode'].append(arch[1:-1])
 520                 elif line.startswith("uses-permission:"):
 521                     perm = re.match(string_pat, line).group(1)
 522                     if perm.startswith("android.permission."):
 523                         perm = perm[19:]
 524                     thisinfo['permissions'].add(perm)
 525                 elif line.startswith("uses-feature:"):
 526                     perm = re.match(string_pat, line).group(1)
 527                     # Filter out this, it's only added with the latest SDK tools and
 528                     # causes problems for lots of apps.
 529                     if perm != "android.hardware.screen.portrait" \
 530                             and perm != "android.hardware.screen.landscape":
 531                         if perm.startswith("android.feature."):
 532                             perm = perm[16:]
 533                         thisinfo['features'].add(perm)
 534
 535             if 'sdkversion' not in thisinfo:
 536                 logging.warn("No SDK version information found in {0}".format(apkfile))
 537                 thisinfo['sdkversion'] = 0
 538
 539             # Check for debuggable apks...
 540             if common.isApkDebuggable(apkfile, config):
 541                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 542
 543             # Get the signature (or md5 of, to be precise)...
 544             logging.debug('Getting signature of {0}'.format(apkfile))
 545             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 546             if not thisinfo['sig']:
 547                 logging.critical("Failed to get apk signature")
 548                 sys.exit(1)
 549
 550             apk = zipfile.ZipFile(apkfile, 'r')
 551
 552             # if an APK has files newer than the system time, suggest updating
 553             # the system clock.  This is useful for offline systems, used for
 554             # signing, which do not have another source of clock sync info. It
 555             # has to be more than 24 hours newer because ZIP/APK files do not
 556             # store timezone info
 557             info = apk.getinfo('AndroidManifest.xml')
 558             dt_obj = datetime(*info.date_time)
 559             checkdt = dt_obj - timedelta(1)
 560             if datetime.today() < checkdt:
 561                 logging.warn('System clock is older than manifest in: '
 562                              + apkfilename + '\nSet clock to that time using:\n'
 563                              + 'sudo date -s "' + str(dt_obj) + '"')
 564
 565             iconfilename = "%s.%s.png" % (
 566                 thisinfo['id'],
 567                 thisinfo['versioncode'])
 568
 569             # Extract the icon file...
 570             empty_densities = []
 571             for density in screen_densities:
 572                 if density not in thisinfo['icons_src']:
 573                     empty_densities.append(density)
 574                     continue
 575                 iconsrc = thisinfo['icons_src'][density]
 576                 icon_dir = get_icon_dir(repodir, density)
 577                 icondest = os.path.join(icon_dir, iconfilename)
 578
 579                 try:
 580                     with open(icondest, 'wb') as f:
 581                         f.write(apk.read(iconsrc))
 582                     thisinfo['icons'][density] = iconfilename
 583
 584                 except:
 585                     logging.warn("Error retrieving icon file")
 586                     del thisinfo['icons'][density]
 587                     del thisinfo['icons_src'][density]
 588                     empty_densities.append(density)
 589
 590             if '-1' in thisinfo['icons_src']:
 591                 iconsrc = thisinfo['icons_src']['-1']
 592                 iconpath = os.path.join(
 593                     get_icon_dir(repodir, '0'), iconfilename)
 594                 with open(iconpath, 'wb') as f:
 595                     f.write(apk.read(iconsrc))
 596                 try:
 597                     im = Image.open(iconpath)
 598                     dpi = px_to_dpi(im.size[0])
 599                     for density in screen_densities:
 600                         if density in thisinfo['icons']:
 601                             break
 602                         if density == screen_densities[-1] or dpi >= int(density):
 603                             thisinfo['icons'][density] = iconfilename
 604                             shutil.move(iconpath,
 605                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 606                             empty_densities.remove(density)
 607                             break
 608                 except Exception, e:
 609                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 610
 611             if thisinfo['icons']:
 612                 thisinfo['icon'] = iconfilename
 613
 614             apk.close()
 615
 616             # First try resizing down to not lose quality
 617             last_density = None
 618             for density in screen_densities:
 619                 if density not in empty_densities:
 620                     last_density = density
 621                     continue
 622                 if last_density is None:
 623                     continue
 624                 logging.debug("Density %s not available, resizing down from %s"
 625                               % (density, last_density))
 626
 627                 last_iconpath = os.path.join(
 628                     get_icon_dir(repodir, last_density), iconfilename)
 629                 iconpath = os.path.join(
 630                     get_icon_dir(repodir, density), iconfilename)
 631                 try:
 632                     im = Image.open(last_iconpath)
 633                 except:
 634                     logging.warn("Invalid image file at %s" % last_iconpath)
 635                     continue
 636
 637                 size = dpi_to_px(density)
 638
 639                 im.thumbnail((size, size), Image.ANTIALIAS)
 640                 im.save(iconpath, "PNG")
 641                 empty_densities.remove(density)
 642
 643             # Then just copy from the highest resolution available
 644             last_density = None
 645             for density in reversed(screen_densities):
 646                 if density not in empty_densities:
 647                     last_density = density
 648                     continue
 649                 if last_density is None:
 650                     continue
 651                 logging.debug("Density %s not available, copying from lower density %s"
 652                               % (density, last_density))
 653
 654                 shutil.copyfile(
 655                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 656                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 657
 658                 empty_densities.remove(density)
 659
 660             for density in screen_densities:
 661                 icon_dir = get_icon_dir(repodir, density)
 662                 icondest = os.path.join(icon_dir, iconfilename)
 663                 resize_icon(icondest, density)
 664
 665             # Copy from icons-mdpi to icons since mdpi is the baseline density
 666             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 667             if os.path.isfile(baseline):
 668                 thisinfo['icons']['0'] = iconfilename
 669                 shutil.copyfile(baseline,
 670                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 671
 672             # Record in known apks, getting the added date at the same time..
 673             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 674             if added:
 675                 thisinfo['added'] = added
 676
 677             apkcache[apkfilename] = thisinfo
 678             cachechanged = True
 679
 680         apks.append(thisinfo)
 681
 682     return apks, cachechanged
 683
 684
 685 repo_pubkey_fingerprint = None
 686
 687
 688 # Generate a certificate fingerprint the same way keytool does it
 689 # (but with slightly different formatting)
 690 def cert_fingerprint(data):
 691     digest = hashlib.sha256(data).digest()
 692     ret = []
 693     ret.append(' '.join("%02X" % ord(b) for b in digest))
 694     return " ".join(ret)
 695
 696
 697 def extract_pubkey():
 698     global repo_pubkey_fingerprint
 699     if 'repo_pubkey' in config:
 700         pubkey = unhexlify(config['repo_pubkey'])
 701     else:
 702         p = FDroidPopen(['keytool', '-exportcert',
 703                          '-alias', config['repo_keyalias'],
 704                          '-keystore', config['keystore'],
 705                          '-storepass:file', config['keystorepassfile']]
 706                         + config['smartcardoptions'], output=False)
 707         if p.returncode != 0 or len(p.output) < 20:
 708             msg = "Failed to get repo pubkey!"
 709             if config['keystore'] == 'NONE':
 710                 msg += ' Is your crypto smartcard plugged in?'
 711             logging.critical(msg)
 712             sys.exit(1)
 713         pubkey = p.output
 714     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 715     return hexlify(pubkey)
 716
 717
 718 def make_index(apps, sortedids, apks, repodir, archive, categories):
 719     """Make a repo index.
 720
 721     :param apps: fully populated apps list
 722     :param apks: full populated apks list
 723     :param repodir: the repo directory
 724     :param archive: True if this is the archive repo, False if it's the
 725                     main one.
 726     :param categories: list of categories
 727     """
 728
 729     doc = Document()
 730
 731     def addElement(name, value, doc, parent):
 732         el = doc.createElement(name)
 733         el.appendChild(doc.createTextNode(value))
 734         parent.appendChild(el)
 735
 736     def addElementNonEmpty(name, value, doc, parent):
 737         if not value:
 738             return
 739         addElement(name, value, doc, parent)
 740
 741     def addElementCDATA(name, value, doc, parent):
 742         el = doc.createElement(name)
 743         el.appendChild(doc.createCDATASection(value))
 744         parent.appendChild(el)
 745
 746     root = doc.createElement("fdroid")
 747     doc.appendChild(root)
 748
 749     repoel = doc.createElement("repo")
 750
 751     if archive:
 752         repoel.setAttribute("name", config['archive_name'])
 753         if config['repo_maxage'] != 0:
 754             repoel.setAttribute("maxage", str(config['repo_maxage']))
 755         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 756         repoel.setAttribute("url", config['archive_url'])
 757         addElement('description', config['archive_description'], doc, repoel)
 758
 759     else:
 760         repoel.setAttribute("name", config['repo_name'])
 761         if config['repo_maxage'] != 0:
 762             repoel.setAttribute("maxage", str(config['repo_maxage']))
 763         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 764         repoel.setAttribute("url", config['repo_url'])
 765         addElement('description', config['repo_description'], doc, repoel)
 766
 767     repoel.setAttribute("version", "14")
 768     repoel.setAttribute("timestamp", str(int(time.time())))
 769
 770     nosigningkey = False
 771     if not options.nosign:
 772         if 'repo_keyalias' not in config:
 773             nosigningkey = True
 774             logging.critical("'repo_keyalias' not found in config.py!")
 775         if 'keystore' not in config:
 776             nosigningkey = True
 777             logging.critical("'keystore' not found in config.py!")
 778         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 779             nosigningkey = True
 780             logging.critical("'keystorepass' not found in config.py!")
 781         if 'keypass' not in config and 'keypassfile' not in config:
 782             nosigningkey = True
 783             logging.critical("'keypass' not found in config.py!")
 784         if not os.path.exists(config['keystore']):
 785             nosigningkey = True
 786             logging.critical("'" + config['keystore'] + "' does not exist!")
 787         if nosigningkey:
 788             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 789             logging.warning("\tfdroid update --create-key")
 790             sys.exit(1)
 791
 792     repoel.setAttribute("pubkey", extract_pubkey())
 793     root.appendChild(repoel)
 794
 795     for appid in sortedids:
 796         app = apps[appid]
 797
 798         if app['Disabled'] is not None:
 799             continue
 800
 801         # Get a list of the apks for this app...
 802         apklist = []
 803         for apk in apks:
 804             if apk['id'] == appid:
 805                 apklist.append(apk)
 806
 807         if len(apklist) == 0:
 808             continue
 809
 810         apel = doc.createElement("application")
 811         apel.setAttribute("id", app['id'])
 812         root.appendChild(apel)
 813
 814         addElement('id', app['id'], doc, apel)
 815         if 'added' in app:
 816             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 817         if 'lastupdated' in app:
 818             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 819         addElement('name', app['Name'], doc, apel)
 820         addElement('summary', app['Summary'], doc, apel)
 821         if app['icon']:
 822             addElement('icon', app['icon'], doc, apel)
 823
 824         def linkres(appid):
 825             if appid in apps:
 826                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 827             raise MetaDataException("Cannot resolve app id " + appid)
 828
 829         addElement('desc',
 830                    metadata.description_html(app['Description'], linkres),
 831                    doc, apel)
 832         addElement('license', app['License'], doc, apel)
 833         if 'Categories' in app and app['Categories']:
 834             addElement('categories', ','.join(app["Categories"]), doc, apel)
 835             # We put the first (primary) category in LAST, which will have
 836             # the desired effect of making clients that only understand one
 837             # category see that one.
 838             addElement('category', app["Categories"][0], doc, apel)
 839         addElement('web', app['Web Site'], doc, apel)
 840         addElement('source', app['Source Code'], doc, apel)
 841         addElement('tracker', app['Issue Tracker'], doc, apel)
 842         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 843         addElementNonEmpty('donate', app['Donate'], doc, apel)
 844         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 845         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 846         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 847
 848         # These elements actually refer to the current version (i.e. which
 849         # one is recommended. They are historically mis-named, and need
 850         # changing, but stay like this for now to support existing clients.
 851         addElement('marketversion', app['Current Version'], doc, apel)
 852         addElement('marketvercode', app['Current Version Code'], doc, apel)
 853
 854         if app['AntiFeatures']:
 855             af = app['AntiFeatures']
 856             if af:
 857                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 858         if app['Provides']:
 859             pv = app['Provides'].split(',')
 860             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 861         if app['Requires Root']:
 862             addElement('requirements', 'root', doc, apel)
 863
 864         # Sort the apk list into version order, just so the web site
 865         # doesn't have to do any work by default...
 866         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 867
 868         # Check for duplicates - they will make the client unhappy...
 869         for i in range(len(apklist) - 1):
 870             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 871                 logging.critical("duplicate versions: '%s' - '%s'" % (
 872                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 873                 sys.exit(1)
 874
 875         current_version_code = 0
 876         current_version_file = None
 877         for apk in apklist:
 878             # find the APK for the "Current Version"
 879             if current_version_code < apk['versioncode']:
 880                 current_version_code = apk['versioncode']
 881             if current_version_code < int(app['Current Version Code']):
 882                 current_version_file = apk['apkname']
 883
 884             apkel = doc.createElement("package")
 885             apel.appendChild(apkel)
 886             addElement('version', apk['version'], doc, apkel)
 887             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 888             addElement('apkname', apk['apkname'], doc, apkel)
 889             if 'srcname' in apk:
 890                 addElement('srcname', apk['srcname'], doc, apkel)
 891             for hash_type in ['sha256']:
 892                 if hash_type not in apk:
 893                     continue
 894                 hashel = doc.createElement("hash")
 895                 hashel.setAttribute("type", hash_type)
 896                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 897                 apkel.appendChild(hashel)
 898             addElement('sig', apk['sig'], doc, apkel)
 899             addElement('size', str(apk['size']), doc, apkel)
 900             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 901             if 'maxsdkversion' in apk:
 902                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 903             if 'added' in apk:
 904                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 905             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 906             if 'nativecode' in apk:
 907                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 908             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 909
 910         if current_version_file is not None \
 911                 and config['make_current_version_link'] \
 912                 and repodir == 'repo':  # only create these
 913             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 914                                     app[config['current_version_name_source']])
 915             apklinkname = sanitized_name + '.apk'
 916             current_version_path = os.path.join(repodir, current_version_file)
 917             if os.path.islink(apklinkname):
 918                 os.remove(apklinkname)
 919             os.symlink(current_version_path, apklinkname)
 920             # also symlink gpg signature, if it exists
 921             for extension in ('.asc', '.sig'):
 922                 sigfile_path = current_version_path + extension
 923                 if os.path.exists(sigfile_path):
 924                     siglinkname = apklinkname + extension
 925                     if os.path.islink(siglinkname):
 926                         os.remove(siglinkname)
 927                     os.symlink(sigfile_path, siglinkname)
 928
 929     if options.pretty:
 930         output = doc.toprettyxml()
 931     else:
 932         output = doc.toxml()
 933
 934     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 935         f.write(output)
 936
 937     if 'repo_keyalias' in config:
 938
 939         if options.nosign:
 940             logging.info("Creating unsigned index in preparation for signing")
 941         else:
 942             logging.info("Creating signed index with this key (SHA256):")
 943             logging.info("%s" % repo_pubkey_fingerprint)
 944
 945         # Create a jar of the index...
 946         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 947         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 948         if p.returncode != 0:
 949             logging.critical("Failed to create {0}".format(jar_output))
 950             sys.exit(1)
 951
 952         # Sign the index...
 953         signed = os.path.join(repodir, 'index.jar')
 954         if options.nosign:
 955             # Remove old signed index if not signing
 956             if os.path.exists(signed):
 957                 os.remove(signed)
 958         else:
 959             args = ['jarsigner', '-keystore', config['keystore'],
 960                     '-storepass:file', config['keystorepassfile'],
 961                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 962                     signed, config['repo_keyalias']]
 963             if config['keystore'] == 'NONE':
 964                 args += config['smartcardoptions']
 965             else:  # smardcards never use -keypass
 966                 args += ['-keypass:file', config['keypassfile']]
 967             p = FDroidPopen(args)
 968             if p.returncode != 0:
 969                 logging.critical("Failed to sign index")
 970                 sys.exit(1)
 971
 972     # Copy the repo icon into the repo directory...
 973     icon_dir = os.path.join(repodir, 'icons')
 974     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 975     shutil.copyfile(config['repo_icon'], iconfilename)
 976
 977     # Write a category list in the repo to allow quick access...
 978     catdata = ''
 979     for cat in categories:
 980         catdata += cat + '\n'
 981     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 982         f.write(catdata)
 983
 984
 985 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 986
 987     for appid, app in apps.iteritems():
 988
 989         if app['Archive Policy']:
 990             keepversions = int(app['Archive Policy'][:-9])
 991         else:
 992             keepversions = defaultkeepversions
 993
 994         def filter_apk_list_sorted(apk_list):
 995             res = []
 996             for apk in apk_list:
 997                 if apk['id'] == appid:
 998                     res.append(apk)
 999
1000             # Sort the apk list by version code. First is highest/newest.
1001             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1002
1003         def move_file(from_dir, to_dir, filename, ignore_missing):
1004             from_path = os.path.join(from_dir, filename)
1005             if ignore_missing and not os.path.exists(from_path):
1006                 return
1007             to_path = os.path.join(to_dir, filename)
1008             shutil.move(from_path, to_path)
1009
1010         if len(apks) > keepversions:
1011             apklist = filter_apk_list_sorted(apks)
1012             # Move back the ones we don't want.
1013             for apk in apklist[keepversions:]:
1014                 logging.info("Moving " + apk['apkname'] + " to archive")
1015                 move_file(repodir, archivedir, apk['apkname'], False)
1016                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1017                 for density in all_screen_densities:
1018                     repo_icon_dir = get_icon_dir(repodir, density)
1019                     archive_icon_dir = get_icon_dir(archivedir, density)
1020                     if density not in apk['icons']:
1021                         continue
1022                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1023                 if 'srcname' in apk:
1024                     move_file(repodir, archivedir, apk['srcname'], False)
1025                 archapks.append(apk)
1026                 apks.remove(apk)
1027         elif len(apks) < keepversions and len(archapks) > 0:
1028             required = keepversions - len(apks)
1029             archapklist = filter_apk_list_sorted(archapks)
1030             # Move forward the ones we want again.
1031             for apk in archapklist[:required]:
1032                 logging.info("Moving " + apk['apkname'] + " from archive")
1033                 move_file(archivedir, repodir, apk['apkname'], False)
1034                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1035                 for density in all_screen_densities:
1036                     repo_icon_dir = get_icon_dir(repodir, density)
1037                     archive_icon_dir = get_icon_dir(archivedir, density)
1038                     if density not in apk['icons']:
1039                         continue
1040                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1041                 if 'srcname' in apk:
1042                     move_file(archivedir, repodir, apk['srcname'], False)
1043                 archapks.remove(apk)
1044                 apks.append(apk)
1045
1046
1047 def add_apks_to_per_app_repos(repodir, apks):
1048     apks_per_app = dict()
1049     for apk in apks:
1050         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1051         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1052         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1053         apks_per_app[apk['id']] = apk
1054
1055         if not os.path.exists(apk['per_app_icons']):
1056             logging.info('Adding new repo for only ' + apk['id'])
1057             os.makedirs(apk['per_app_icons'])
1058
1059         apkpath = os.path.join(repodir, apk['apkname'])
1060         shutil.copy(apkpath, apk['per_app_repo'])
1061         apksigpath = apkpath + '.sig'
1062         if os.path.exists(apksigpath):
1063             shutil.copy(apksigpath, apk['per_app_repo'])
1064         apkascpath = apkpath + '.asc'
1065         if os.path.exists(apkascpath):
1066             shutil.copy(apkascpath, apk['per_app_repo'])
1067
1068
1069 config = None
1070 options = None
1071
1072
1073 def main():
1074
1075     global config, options
1076
1077     # Parse command line...
1078     parser = ArgumentParser()
1079     common.setup_global_opts(parser)
1080     parser.add_argument("--create-key", action="store_true", default=False,
1081                         help="Create a repo signing key in a keystore")
1082     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1083                         help="Create skeleton metadata files that are missing")
1084     parser.add_argument("--delete-unknown", action="store_true", default=False,
1085                         help="Delete APKs without metadata from the repo")
1086     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1087                         help="Report on build data status")
1088     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1089                         help="Interactively ask about things that need updating.")
1090     parser.add_argument("-I", "--icons", action="store_true", default=False,
1091                         help="Resize all the icons exceeding the max pixel size and exit")
1092     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1093                         help="Specify editor to use in interactive mode. Default " +
1094                         "is /etc/alternatives/editor")
1095     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1096                         help="Update the wiki")
1097     parser.add_argument("--pretty", action="store_true", default=False,
1098                         help="Produce human-readable index.xml")
1099     parser.add_argument("--clean", action="store_true", default=False,
1100                         help="Clean update - don't uses caches, reprocess all apks")
1101     parser.add_argument("--nosign", action="store_true", default=False,
1102                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1103     options = parser.parse_args()
1104
1105     config = common.read_config(options)
1106
1107     repodirs = ['repo']
1108     if config['archive_older'] != 0:
1109         repodirs.append('archive')
1110         if not os.path.exists('archive'):
1111             os.mkdir('archive')
1112
1113     if options.icons:
1114         resize_all_icons(repodirs)
1115         sys.exit(0)
1116
1117     # check that icons exist now, rather than fail at the end of `fdroid update`
1118     for k in ['repo_icon', 'archive_icon']:
1119         if k in config:
1120             if not os.path.exists(config[k]):
1121                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1122                 sys.exit(1)
1123
1124     # if the user asks to create a keystore, do it now, reusing whatever it can
1125     if options.create_key:
1126         if os.path.exists(config['keystore']):
1127             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1128             logging.critical("\t'" + config['keystore'] + "'")
1129             sys.exit(1)
1130
1131         if 'repo_keyalias' not in config:
1132             config['repo_keyalias'] = socket.getfqdn()
1133             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1134         if 'keydname' not in config:
1135             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1136             common.write_to_config(config, 'keydname', config['keydname'])
1137         if 'keystore' not in config:
1138             config['keystore'] = common.default_config.keystore
1139             common.write_to_config(config, 'keystore', config['keystore'])
1140
1141         password = common.genpassword()
1142         if 'keystorepass' not in config:
1143             config['keystorepass'] = password
1144             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1145         if 'keypass' not in config:
1146             config['keypass'] = password
1147             common.write_to_config(config, 'keypass', config['keypass'])
1148         common.genkeystore(config)
1149
1150     # Get all apps...
1151     apps = metadata.read_metadata()
1152
1153     # Generate a list of categories...
1154     categories = set()
1155     for app in apps.itervalues():
1156         categories.update(app['Categories'])
1157
1158     # Read known apks data (will be updated and written back when we've finished)
1159     knownapks = common.KnownApks()
1160
1161     # Gather information about all the apk files in the repo directory, using
1162     # cached data if possible.
1163     apkcachefile = os.path.join('tmp', 'apkcache')
1164     if not options.clean and os.path.exists(apkcachefile):
1165         with open(apkcachefile, 'rb') as cf:
1166             apkcache = pickle.load(cf)
1167     else:
1168         apkcache = {}
1169
1170     delete_disabled_builds(apps, apkcache, repodirs)
1171
1172     # Scan all apks in the main repo
1173     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1174
1175     # Generate warnings for apk's with no metadata (or create skeleton
1176     # metadata files, if requested on the command line)
1177     newmetadata = False
1178     for apk in apks:
1179         if apk['id'] not in apps:
1180             if options.create_metadata:
1181                 if 'name' not in apk:
1182                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1183                     continue
1184                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1185                 f.write("License:Unknown\n")
1186                 f.write("Web Site:\n")
1187                 f.write("Source Code:\n")
1188                 f.write("Issue Tracker:\n")
1189                 f.write("Changelog:\n")
1190                 f.write("Summary:" + apk['name'] + "\n")
1191                 f.write("Description:\n")
1192                 f.write(apk['name'] + "\n")
1193                 f.write(".\n")
1194                 f.close()
1195                 logging.info("Generated skeleton metadata for " + apk['id'])
1196                 newmetadata = True
1197             else:
1198                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1199                 if options.delete_unknown:
1200                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1201                     rmf = os.path.join(repodirs[0], apk['apkname'])
1202                     if not os.path.exists(rmf):
1203                         logging.error("Could not find {0} to remove it".format(rmf))
1204                     else:
1205                         os.remove(rmf)
1206                 else:
1207                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1208
1209     # update the metadata with the newly created ones included
1210     if newmetadata:
1211         apps = metadata.read_metadata()
1212
1213     # Scan the archive repo for apks as well
1214     if len(repodirs) > 1:
1215         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1216         if cc:
1217             cachechanged = True
1218     else:
1219         archapks = []
1220
1221     # Some information from the apks needs to be applied up to the application
1222     # level. When doing this, we use the info from the most recent version's apk.
1223     # We deal with figuring out when the app was added and last updated at the
1224     # same time.
1225     for appid, app in apps.iteritems():
1226         bestver = 0
1227         added = None
1228         lastupdated = None
1229         for apk in apks + archapks:
1230             if apk['id'] == appid:
1231                 if apk['versioncode'] > bestver:
1232                     bestver = apk['versioncode']
1233                     bestapk = apk
1234
1235                 if 'added' in apk:
1236                     if not added or apk['added'] < added:
1237                         added = apk['added']
1238                     if not lastupdated or apk['added'] > lastupdated:
1239                         lastupdated = apk['added']
1240
1241         if added:
1242             app['added'] = added
1243         else:
1244             logging.debug("Don't know when " + appid + " was added")
1245         if lastupdated:
1246             app['lastupdated'] = lastupdated
1247         else:
1248             logging.debug("Don't know when " + appid + " was last updated")
1249
1250         if bestver == 0:
1251             if app['Name'] is None:
1252                 app['Name'] = app['Auto Name'] or appid
1253             app['icon'] = None
1254             logging.debug("Application " + appid + " has no packages")
1255         else:
1256             if app['Name'] is None:
1257                 app['Name'] = bestapk['name']
1258             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1259
1260     # Sort the app list by name, then the web site doesn't have to by default.
1261     # (we had to wait until we'd scanned the apks to do this, because mostly the
1262     # name comes from there!)
1263     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1264
1265     # APKs are placed into multiple repos based on the app package, providing
1266     # per-app subscription feeds for nightly builds and things like it
1267     if config['per_app_repos']:
1268         add_apks_to_per_app_repos(repodirs[0], apks)
1269         for appid, app in apps.iteritems():
1270             repodir = os.path.join(appid, 'fdroid', 'repo')
1271             appdict = dict()
1272             appdict[appid] = app
1273             if os.path.isdir(repodir):
1274                 make_index(appdict, [appid], apks, repodir, False, categories)
1275             else:
1276                 logging.info('Skipping index generation for ' + appid)
1277         return
1278
1279     if len(repodirs) > 1:
1280         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1281
1282     # Make the index for the main repo...
1283     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1284
1285     # If there's an archive repo,  make the index for it. We already scanned it
1286     # earlier on.
1287     if len(repodirs) > 1:
1288         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1289
1290     if config['update_stats']:
1291
1292         # Update known apks info...
1293         knownapks.writeifchanged()
1294
1295         # Generate latest apps data for widget
1296         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1297             data = ''
1298             for line in file(os.path.join('stats', 'latestapps.txt')):
1299                 appid = line.rstrip()
1300                 data += appid + "\t"
1301                 app = apps[appid]
1302                 data += app['Name'] + "\t"
1303                 if app['icon'] is not None:
1304                     data += app['icon'] + "\t"
1305                 data += app['License'] + "\n"
1306             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1307                 f.write(data)
1308
1309     if cachechanged:
1310         with open(apkcachefile, 'wb') as cf:
1311             pickle.dump(apkcache, cf)
1312
1313     # Update the wiki...
1314     if options.wiki:
1315         update_wiki(apps, sortedids, apks + archapks)
1316
1317     logging.info("Finished.")
1318
1319 if __name__ == "__main__":
1320     main()