fdroidserver/server.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # server.py - part of the FDroid server tools
   5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import hashlib
  22 import os
  23 import subprocess
  24 from optparse import OptionParser
  25 import logging
  26 import common
  27
  28 config = None
  29 options = None
  30
  31 def update_awsbucket(repo_section):
  32     '''
  33     Upload the contents of the directory `repo_section` (including
  34     subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
  35     bucket will first be deleted.
  36
  37     Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
  38     '''
  39
  40     import libcloud.security
  41     libcloud.security.VERIFY_SSL_CERT = True
  42     from libcloud.storage.types import Provider, ContainerDoesNotExistError
  43     from libcloud.storage.providers import get_driver
  44
  45     if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
  46         logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
  47         sys.exit(1)
  48     awsbucket = config['awsbucket']
  49
  50     cls = get_driver(Provider.S3)
  51     driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
  52     try:
  53         container = driver.get_container(container_name=awsbucket)
  54     except ContainerDoesNotExistError:
  55         container = driver.create_container(container_name=awsbucket)
  56         logging.info('Created new container "' + container.name + '"')
  57
  58     upload_dir = 'fdroid/' + repo_section
  59     objs = dict()
  60     for obj in container.list_objects():
  61         if obj.name.startswith(upload_dir + '/'):
  62             objs[obj.name] = obj
  63
  64     for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
  65         for name in files:
  66             upload = False
  67             file_to_upload = os.path.join(root, name)
  68             object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
  69             if not object_name in objs:
  70                 upload = True
  71             else:
  72                 obj = objs.pop(object_name)
  73                 if obj.size != os.path.getsize(file_to_upload):
  74                     upload = True
  75                 else:
  76                     # if the sizes match, then compare by MD5
  77                     md5 = hashlib.md5()
  78                     with open(file_to_upload, 'rb') as f:
  79                         while True:
  80                             data = f.read(8192)
  81                             if not data:
  82                                 break
  83                             md5.update(data)
  84                     if obj.hash != md5.hexdigest():
  85                         s3url = 's3://' + awsbucket + '/' + obj.name
  86                         logging.info(' deleting ' + s3url)
  87                         if not driver.delete_object(obj):
  88                             logging.warn('Could not delete ' + s3url)
  89                         upload = True
  90
  91             if upload:
  92                 if options.verbose:
  93                     logging.info(' uploading "' + file_to_upload + '"...')
  94                 extra = {'acl': 'public-read'}
  95                 if file_to_upload.endswith('.sig'):
  96                     extra['content_type'] = 'application/pgp-signature'
  97                 elif file_to_upload.endswith('.asc'):
  98                     extra['content_type'] = 'application/pgp-signature'
  99                 logging.info(' uploading ' + os.path.relpath(file_to_upload)
 100                              + ' to s3://' + awsbucket + '/' + object_name)
 101                 obj = driver.upload_object(file_path=file_to_upload,
 102                                            container=container,
 103                                            object_name=object_name,
 104                                            verify_hash=False,
 105                                            extra=extra)
 106     # delete the remnants in the bucket, they do not exist locally
 107     while objs:
 108         object_name, obj = objs.popitem()
 109         s3url = 's3://' + awsbucket + '/' + object_name
 110         if object_name.startswith(upload_dir):
 111             logging.warn(' deleting ' + s3url)
 112             driver.delete_object(obj)
 113         else:
 114             logging.info(' skipping ' + s3url)
 115
 116 def update_serverwebroot(repo_section):
 117     rsyncargs = ['rsync', '-u', '-r', '--delete']
 118     if options.verbose:
 119         rsyncargs += ['--verbose']
 120     if options.quiet:
 121         rsyncargs += ['--quiet']
 122     index = os.path.join(repo_section, 'index.xml')
 123     indexjar = os.path.join(repo_section, 'index.jar')
 124     # serverwebroot is guaranteed to have a trailing slash in common.py
 125     if subprocess.call(rsyncargs +
 126                        ['--exclude', index, '--exclude', indexjar,
 127                         repo_section, config['serverwebroot']]) != 0:
 128         sys.exit(1)
 129     if subprocess.call(rsyncargs +
 130                        [index, config['serverwebroot'] + repo_section]) != 0:
 131         sys.exit(1)
 132     if subprocess.call(rsyncargs +
 133                        [indexjar, config['serverwebroot'] + repo_section]) != 0:
 134         sys.exit(1)
 135
 136 def main():
 137     global config, options
 138
 139     # Parse command line...
 140     parser = OptionParser()
 141     parser.add_option("-v", "--verbose", action="store_true", default=False,
 142                       help="Spew out even more information than normal")
 143     parser.add_option("-q", "--quiet", action="store_true", default=False,
 144                       help="Restrict output to warnings and errors")
 145     (options, args) = parser.parse_args()
 146
 147     config = common.read_config(options)
 148
 149     if len(args) != 1:
 150         logging.critical("Specify a single command")
 151         sys.exit(1)
 152
 153     if args[0] != 'init' and args[0] != 'update':
 154         logging.critical("The only commands currently supported are 'init' and 'update'")
 155         sys.exit(1)
 156
 157     if config.get('nonstandardwebroot') is True:
 158         standardwebroot = False
 159     else:
 160         standardwebroot = True
 161
 162     if config.get('serverwebroot'):
 163         serverwebroot = config['serverwebroot']
 164         host, fdroiddir = serverwebroot.rstrip('/').split(':')
 165         serverrepobase = os.path.basename(fdroiddir)
 166         if serverrepobase != 'fdroid' and standardwebroot:
 167             logging.error('serverwebroot does not end with "fdroid", '
 168                           + 'perhaps you meant one of these:\n\t'
 169                           + serverwebroot.rstrip('/') + '/fdroid\n\t'
 170                           + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
 171             sys.exit(1)
 172     elif not config.get('awsbucket'):
 173         logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.')
 174         sys.exit(1)
 175
 176     repo_sections = ['repo']
 177     if config['archive_older'] != 0:
 178         repo_sections.append('archive')
 179
 180     if args[0] == 'init':
 181         if config.get('serverwebroot'):
 182             sshargs = ['ssh']
 183             if options.quiet:
 184                 sshargs += ['-q']
 185             for repo_section in repo_sections:
 186                 cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section]
 187                 if options.verbose:
 188                     # ssh -v produces different output than rsync -v, so this
 189                     # simulates rsync -v
 190                     logging.info(' '.join(cmd))
 191                 if subprocess.call(cmd) != 0:
 192                     sys.exit(1)
 193     elif args[0] == 'update':
 194         for repo_section in repo_sections:
 195             if config.get('serverwebroot'):
 196                 update_serverwebroot(repo_section)
 197             if config.get('awsbucket'):
 198                 update_awsbucket(repo_section)
 199
 200     sys.exit(0)
 201
 202 if __name__ == "__main__":
 203     main()