update.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import subprocess
  25 import re
  26 import zipfile
  27 import hashlib
  28 from xml.dom.minidom import Document
  29 from optparse import OptionParser
  30 import time
  31
  32 def main():
  33
  34     # Read configuration...
  35     execfile('config.py', globals())
  36
  37     import common
  38
  39     # Parse command line...
  40     parser = OptionParser()
  41     parser.add_option("-c", "--createmeta", action="store_true", default=False,
  42                       help="Create skeleton metadata files that are missing")
  43     parser.add_option("-v", "--verbose", action="store_true", default=False,
  44                       help="Spew out even more information than normal")
  45     parser.add_option("-q", "--quiet", action="store_true", default=False,
  46                       help="No output, except for warnings and errors")
  47     parser.add_option("-b", "--buildreport", action="store_true", default=False,
  48                       help="Report on build data status")
  49     parser.add_option("-i", "--interactive", default=False, action="store_true",
  50                       help="Interactively ask about things that need updating.")
  51     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
  52                       help="Specify editor to use in interactive mode. Default "+
  53                           "is /etc/alternatives/editor")
  54     parser.add_option("", "--pretty", action="store_true", default=False,
  55                       help="Produce human-readable index.xml")
  56     (options, args) = parser.parse_args()
  57
  58
  59     icon_dir=os.path.join('repo','icons')
  60
  61     # Delete and re-create the icon directory...
  62     if os.path.exists(icon_dir):
  63         shutil.rmtree(icon_dir)
  64     os.mkdir(icon_dir)
  65
  66     warnings = 0
  67
  68     # Get all apps...
  69     apps = common.read_metadata(verbose=options.verbose)
  70
  71     # Generate a list of categories...
  72     categories = []
  73     for app in apps:
  74         if app['Category'] not in categories:
  75             categories.append(app['Category'])
  76
  77     # Gather information about all the apk files in the repo directory...
  78     apks = []
  79     for apkfile in glob.glob(os.path.join('repo','*.apk')):
  80
  81         apkfilename = apkfile[5:]
  82         if apkfilename.find(' ') != -1:
  83             print "No spaces in APK filenames!"
  84             sys.exit(1)
  85         srcfilename = apkfilename[:-4] + "_src.tar.gz"
  86
  87         if not options.quiet:
  88             print "Processing " + apkfilename
  89         thisinfo = {}
  90         thisinfo['apkname'] = apkfilename
  91         if os.path.exists(os.path.join('repo', srcfilename)):
  92             thisinfo['srcname'] = srcfilename
  93         thisinfo['size'] = os.path.getsize(apkfile)
  94         thisinfo['permissions'] = []
  95         thisinfo['features'] = []
  96         p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
  97                               'dump', 'badging', apkfile],
  98                              stdout=subprocess.PIPE)
  99         output = p.communicate()[0]
 100         if options.verbose:
 101             print output
 102         if p.returncode != 0:
 103             print "ERROR: Failed to get apk information"
 104             sys.exit(1)
 105         for line in output.splitlines():
 106             if line.startswith("package:"):
 107                 pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 108                 thisinfo['id'] = re.match(pat, line).group(1)
 109                 pat = re.compile(".*versionCode='([0-9]*)'.*")
 110                 thisinfo['versioncode'] = int(re.match(pat, line).group(1))
 111                 pat = re.compile(".*versionName='([^']*)'.*")
 112                 thisinfo['version'] = re.match(pat, line).group(1)
 113             if line.startswith("application:"):
 114                 pat = re.compile(".*label='([^']*)'.*")
 115                 thisinfo['name'] = re.match(pat, line).group(1)
 116                 pat = re.compile(".*icon='([^']*)'.*")
 117                 thisinfo['iconsrc'] = re.match(pat, line).group(1)
 118             if line.startswith("sdkVersion:"):
 119                 pat = re.compile(".*'([0-9]*)'.*")
 120                 thisinfo['sdkversion'] = re.match(pat, line).group(1)
 121             if line.startswith("native-code:"):
 122                 pat = re.compile(".*'([^']*)'.*")
 123                 thisinfo['nativecode'] = re.match(pat, line).group(1)
 124             if line.startswith("uses-permission:"):
 125                 pat = re.compile(".*'([^']*)'.*")
 126                 perm = re.match(pat, line).group(1)
 127                 if perm.startswith("android.permission."):
 128                     perm = perm[19:]
 129                 thisinfo['permissions'].append(perm)
 130             if line.startswith("uses-feature:"):
 131                 pat = re.compile(".*'([^']*)'.*")
 132                 perm = re.match(pat, line).group(1)
 133                 #Filter out this, it's only added with the latest SDK tools and
 134                 #causes problems for lots of apps.
 135                 if (perm != "android.hardware.screen.portrait" and
 136                     perm != "android.hardware.screen.landscape"):
 137                     if perm.startswith("android.feature."):
 138                         perm = perm[16:]
 139                     thisinfo['features'].append(perm)
 140
 141         if not thisinfo.has_key('sdkversion'):
 142             print "  WARNING: no SDK version information found"
 143             thisinfo['sdkversion'] = 0
 144
 145         # Calculate the md5 and sha256...
 146         m = hashlib.md5()
 147         sha = hashlib.sha256()
 148         f = open(apkfile, 'rb')
 149         while True:
 150             t = f.read(1024)
 151             if len(t) == 0:
 152                 break
 153             m.update(t)
 154             sha.update(t)
 155         thisinfo['md5'] = m.hexdigest()
 156         thisinfo['sha256'] = sha.hexdigest()
 157         f.close()
 158
 159         # Get the signature (or md5 of, to be precise)...
 160         p = subprocess.Popen(['java', 'getsig',
 161                               os.path.join(os.getcwd(), apkfile)],
 162                              cwd=os.path.join(sys.path[0], 'getsig'),
 163                              stdout=subprocess.PIPE)
 164         output = p.communicate()[0]
 165         if options.verbose:
 166             print output
 167         if p.returncode != 0 or not output.startswith('Result:'):
 168             print "ERROR: Failed to get apk signature"
 169             sys.exit(1)
 170         thisinfo['sig'] = output[7:].strip()
 171
 172         # Extract the icon file...
 173         apk = zipfile.ZipFile(apkfile, 'r')
 174         thisinfo['icon'] = (thisinfo['id'] + '.' +
 175             str(thisinfo['versioncode']) + '.png')
 176         iconfilename = os.path.join(icon_dir, thisinfo['icon'])
 177         try:
 178             iconfile = open(iconfilename, 'wb')
 179             iconfile.write(apk.read(thisinfo['iconsrc']))
 180             iconfile.close()
 181         except:
 182             print "WARNING: Error retrieving icon file"
 183             warnings += 1
 184         apk.close()
 185
 186         apks.append(thisinfo)
 187
 188     # Some information from the apks needs to be applied up to the application
 189     # level. When doing this, we use the info from the most recent version's apk.
 190     for app in apps:
 191         bestver = 0
 192         for apk in apks:
 193             if apk['id'] == app['id']:
 194                 if apk['versioncode'] > bestver:
 195                     bestver = apk['versioncode']
 196                     bestapk = apk
 197
 198         if bestver == 0:
 199             if app['Name'] is None:
 200                 app['Name'] = app['id']
 201             app['icon'] = ''
 202             if app['Disabled'] is None:
 203                 print "WARNING: Application " + app['id'] + " has no packages"
 204         else:
 205             if app['Name'] is None:
 206                 app['Name'] = bestapk['name']
 207             app['icon'] = bestapk['icon']
 208
 209     # Generate warnings for apk's with no metadata (or create skeleton
 210     # metadata files, if requested on the command line)
 211     for apk in apks:
 212         found = False
 213         for app in apps:
 214             if app['id'] == apk['id']:
 215                 found = True
 216                 break
 217         if not found:
 218             if options.createmeta:
 219                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 220                 f.write("License:Unknown\n")
 221                 f.write("Web Site:\n")
 222                 f.write("Source Code:\n")
 223                 f.write("Issue Tracker:\n")
 224                 f.write("Summary:" + apk['name'] + "\n")
 225                 f.write("Description:\n")
 226                 f.write(apk['name'] + "\n")
 227                 f.write(".\n")
 228                 f.close()
 229                 print "Generated skeleton metadata for " + apk['id']
 230             else:
 231                 print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
 232                 print "       " + apk['name'] + " - " + apk['version']
 233
 234     #Sort the app list by name, then the web site doesn't have to by default:
 235     apps = sorted(apps, key=lambda app: app['Name'].upper())
 236
 237     # Create the index
 238     doc = Document()
 239
 240     def addElement(name, value, doc, parent):
 241         el = doc.createElement(name)
 242         el.appendChild(doc.createTextNode(value))
 243         parent.appendChild(el)
 244
 245     root = doc.createElement("fdroid")
 246     doc.appendChild(root)
 247
 248     repoel = doc.createElement("repo")
 249     repoel.setAttribute("name", repo_name)
 250     repoel.setAttribute("icon", os.path.basename(repo_icon))
 251     repoel.setAttribute("url", repo_url)
 252
 253     if repo_keyalias != None:
 254
 255         # Generate a certificate fingerprint the same way keytool does it
 256         # (but with slightly different formatting)
 257         def cert_fingerprint(data):
 258             digest = hashlib.sha1(data).digest()
 259             ret = []
 260             for i in range(4):
 261                 ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
 262             return " ".join(ret)
 263
 264         def extract_pubkey():
 265             p = subprocess.Popen(['keytool', '-exportcert',
 266                                   '-alias', repo_keyalias,
 267                                   '-keystore', keystore,
 268                                   '-storepass', keystorepass],
 269                                  stdout=subprocess.PIPE)
 270             cert = p.communicate()[0]
 271             if p.returncode != 0:
 272                 print "ERROR: Failed to get repo pubkey"
 273                 sys.exit(1)
 274             global repo_pubkey_fingerprint
 275             repo_pubkey_fingerprint = cert_fingerprint(cert)
 276             return "".join("%02x" % ord(b) for b in cert)
 277
 278         repoel.setAttribute("pubkey", extract_pubkey())
 279
 280     addElement('description', repo_description, doc, repoel)
 281     root.appendChild(repoel)
 282
 283     apps_inrepo = 0
 284     apps_disabled = 0
 285     apps_nopkg = 0
 286
 287     for app in apps:
 288
 289         if app['Disabled'] is None:
 290
 291             # Get a list of the apks for this app...
 292             gotcurrentver = False
 293             apklist = []
 294             for apk in apks:
 295                 if apk['id'] == app['id']:
 296                     if str(apk['versioncode']) == app['Current Version Code']:
 297                         gotcurrentver = True
 298                     apklist.append(apk)
 299
 300             if len(apklist) == 0:
 301                 apps_nopkg += 1
 302             else:
 303                 apps_inrepo += 1
 304                 apel = doc.createElement("application")
 305                 apel.setAttribute("id", app['id'])
 306                 root.appendChild(apel)
 307
 308                 addElement('id', app['id'], doc, apel)
 309                 addElement('name', app['Name'], doc, apel)
 310                 addElement('summary', app['Summary'], doc, apel)
 311                 addElement('icon', app['icon'], doc, apel)
 312                 addElement('description',
 313                         common.parse_description(app['Description']), doc, apel)
 314                 addElement('license', app['License'], doc, apel)
 315                 if 'Category' in app:
 316                     addElement('category', app['Category'], doc, apel)
 317                 addElement('web', app['Web Site'], doc, apel)
 318                 addElement('source', app['Source Code'], doc, apel)
 319                 addElement('tracker', app['Issue Tracker'], doc, apel)
 320                 if app['Donate'] != None:
 321                     addElement('donate', app['Donate'], doc, apel)
 322
 323                 # These elements actually refer to the current version (i.e. which
 324                 # one is recommended. They are historically mis-named, and need
 325                 # changing, but stay like this for now to support existing clients.
 326                 addElement('marketversion', app['Current Version'], doc, apel)
 327                 addElement('marketvercode', app['Current Version Code'], doc, apel)
 328
 329                 if not (app['AntiFeatures'] is None):
 330                     addElement('antifeatures', app['AntiFeatures'], doc, apel)
 331                 if app['Requires Root']:
 332                     addElement('requirements', 'root', doc, apel)
 333
 334                 # Sort the apk list into version order, just so the web site
 335                 # doesn't have to do any work by default...
 336                 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 337
 338                 # Check for duplicates - they will make the client unhappy...
 339                 for i in range(len(apklist) - 1):
 340                     if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
 341                         print "ERROR - duplicate versions"
 342                         print apklist[i]['apkname']
 343                         print apklist[i+1]['apkname']
 344                         sys.exit(1)
 345
 346                 for apk in apklist:
 347                     apkel = doc.createElement("package")
 348                     apel.appendChild(apkel)
 349                     addElement('version', apk['version'], doc, apkel)
 350                     addElement('versioncode', str(apk['versioncode']), doc, apkel)
 351                     addElement('apkname', apk['apkname'], doc, apkel)
 352                     if apk.has_key('srcname'):
 353                         addElement('srcname', apk['srcname'], doc, apkel)
 354                     for hash_type in ('sha256', 'md5'):
 355                         if not hash_type in apk:
 356                             continue
 357                         hashel = doc.createElement("hash")
 358                         hashel.setAttribute("type", hash_type)
 359                         hashel.appendChild(doc.createTextNode(apk[hash_type]))
 360                         apkel.appendChild(hashel)
 361                     addElement('sig', apk['sig'], doc, apkel)
 362                     addElement('size', str(apk['size']), doc, apkel)
 363                     addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 364                     perms = ""
 365                     for p in apk['permissions']:
 366                         if len(perms) > 0:
 367                             perms += ","
 368                         perms += p
 369                     if len(perms) > 0:
 370                         addElement('permissions', perms, doc, apkel)
 371                     features = ""
 372                     for f in apk['features']:
 373                         if len(features) > 0:
 374                             features += ","
 375                         features += f
 376                     if len(features) > 0:
 377                         addElement('features', features, doc, apkel)
 378
 379             if options.buildreport:
 380                 if len(app['builds']) == 0:
 381                     print ("WARNING: No builds defined for " + app['id'] +
 382                             " Source: " + app['Source Code'])
 383                     warnings += 1
 384                 else:
 385                     if app['Current Version Code'] != '0':
 386                         gotbuild = False
 387                         for build in app['builds']:
 388                             if build['vercode'] == app['Current Version Code']:
 389                                 gotbuild = True
 390                         if not gotbuild:
 391                             print ("WARNING: No build data for current version of "
 392                                     + app['id'] + " (" + app['Current Version']
 393                                     + ") " + app['Source Code'])
 394                             warnings += 1
 395
 396             # If we don't have the current version, check if there is a build
 397             # with a commit ID starting with '!' - this means we can't build it
 398             # for some reason, and don't want hassling about it...
 399             if not gotcurrentver and app['Current Version Code'] != '0':
 400                 for build in app['builds']:
 401                     if build['vercode'] == app['Current Version Code']:
 402                         gotcurrentver = True
 403
 404             # Output a message of harassment if we don't have the current version:
 405             if not gotcurrentver and app['Current Version Code'] != '0':
 406                 addr = app['Source Code']
 407                 print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name']
 408                 print "         (" + app['id'] + ") " + addr
 409                 warnings += 1
 410                 if options.verbose:
 411                     # A bit of extra debug info, basically for diagnosing
 412                     # app developer mistakes:
 413                     print "         Current vercode:" + app['Current Version Code']
 414                     print "         Got:"
 415                     for apk in apks:
 416                         if apk['id'] == app['id']:
 417                             print "           " + str(apk['versioncode']) + " - " + apk['version']
 418                 if options.interactive:
 419                     print "Build data out of date for " + app['id']
 420                     while True:
 421                         answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower()
 422                         if answer == 'i':
 423                             break
 424                         elif answer == 'e':
 425                             subprocess.call([options.editor,
 426                                 os.path.join('metadata',
 427                                 app['id'] + '.txt')])
 428                             break
 429                         elif answer == 'q':
 430                             sys.exit(0)
 431         else:
 432             apps_disabled += 1
 433
 434     of = open(os.path.join('repo','index.xml'), 'wb')
 435     if options.pretty:
 436         output = doc.toprettyxml()
 437     else:
 438         output = doc.toxml()
 439     of.write(output)
 440     of.close()
 441
 442     if repo_keyalias != None:
 443
 444         if not options.quiet:
 445             print "Creating signed index."
 446             print "Key fingerprint:", repo_pubkey_fingerprint
 447
 448         #Create a jar of the index...
 449         p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
 450             cwd='repo', stdout=subprocess.PIPE)
 451         output = p.communicate()[0]
 452         if options.verbose:
 453             print output
 454         if p.returncode != 0:
 455             print "ERROR: Failed to create jar file"
 456             sys.exit(1)
 457
 458         # Sign the index...
 459         p = subprocess.Popen(['jarsigner', '-keystore', keystore,
 460             '-storepass', keystorepass, '-keypass', keypass,
 461             os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE)
 462         output = p.communicate()[0]
 463         if p.returncode != 0:
 464             print "Failed to sign index"
 465             print output
 466             sys.exit(1)
 467         if options.verbose:
 468             print output
 469
 470     # Copy the repo icon into the repo directory...
 471     iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
 472     shutil.copyfile(repo_icon, iconfilename)
 473
 474     # Write a category list in the repo to allow quick access...
 475     catdata = ''
 476     for cat in categories:
 477         catdata += cat + '\n'
 478     f = open('repo/categories.txt', 'w')
 479     f.write(catdata)
 480     f.close()
 481
 482     # Update known apks info...
 483     knownapks = common.KnownApks()
 484     for apk in apks:
 485         knownapks.recordapk(apk['apkname'], apk['id'])
 486     knownapks.writeifchanged()
 487
 488     # Generate latest apps data for widget
 489     data = ''
 490     for line in file(os.path.join('stats', 'latestapps.txt')):
 491         appid = line.rstrip()
 492         data += appid + "\t"
 493         for app in apps:
 494             if app['id'] == appid:
 495                 data += app['Name'] + "\t"
 496                 data += app['icon'] + "\t"
 497                 data += app['License'] + "\n"
 498                 break
 499     f = open('repo/latestapps.dat', 'w')
 500     f.write(data)
 501     f.close()
 502
 503
 504
 505     print "Finished."
 506     print str(apps_inrepo) + " apps in repo"
 507     print str(apps_disabled) + " disabled"
 508     print str(apps_nopkg) + " with no packages"
 509     print str(warnings) + " warnings"
 510
 511 if __name__ == "__main__":
 512     main()
 513