# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
+import hashlib
import os
import subprocess
from optparse import OptionParser
config = None
options = None
+def update_awsbucket(repo_section):
+ '''
+ Upload the contents of the directory `repo_section` (including
+ subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
+ bucket will first be deleted.
-def main():
+ Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
+ '''
+
+ import libcloud.security
+ libcloud.security.VERIFY_SSL_CERT = True
+ from libcloud.storage.types import Provider, ContainerDoesNotExistError
+ from libcloud.storage.providers import get_driver
+
+ if 'awsaccesskeyid' not in config or 'awssecretkey' not in config:
+ logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
+ sys.exit(1)
+ awsbucket = config['awsbucket']
+
+ cls = get_driver(Provider.S3)
+ driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
+ try:
+ container = driver.get_container(container_name=awsbucket)
+ except ContainerDoesNotExistError:
+ container = driver.create_container(container_name=awsbucket)
+ logging.info('Created new container "' + container.name + '"')
+
+ upload_dir = 'fdroid/' + repo_section
+ objs = dict()
+ for obj in container.list_objects():
+ if obj.name.startswith(upload_dir + '/'):
+ objs[obj.name] = obj
+
+ for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
+ for name in files:
+ upload = False
+ file_to_upload = os.path.join(root, name)
+ object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
+ if not object_name in objs:
+ upload = True
+ else:
+ obj = objs.pop(object_name)
+ if obj.size != os.path.getsize(file_to_upload):
+ upload = True
+ else:
+ # if the sizes match, then compare by MD5
+ md5 = hashlib.md5()
+ with open(file_to_upload, 'rb') as f:
+ while True:
+ data = f.read(8192)
+ if not data:
+ break
+ md5.update(data)
+ if obj.hash != md5.hexdigest():
+ s3url = 's3://' + awsbucket + '/' + obj.name
+ logging.info(' deleting ' + s3url)
+ if not driver.delete_object(obj):
+ logging.warn('Could not delete ' + s3url)
+ upload = True
+
+ if upload:
+ if options.verbose:
+ logging.info(' uploading "' + file_to_upload + '"...')
+ extra = { 'acl': 'public-read' }
+ if file_to_upload.endswith('.sig'):
+ extra['content_type'] = 'application/pgp-signature'
+ elif file_to_upload.endswith('.asc'):
+ extra['content_type'] = 'application/pgp-signature'
+ logging.info(' uploading ' + os.path.relpath(file_to_upload)
+ + ' to s3://' + awsbucket + '/' + obj.name)
+ obj = driver.upload_object(file_path=file_to_upload,
+ container=container,
+ object_name=object_name,
+ verify_hash=False,
+ extra=extra)
+ # delete the remnants in the bucket, they do not exist locally
+ while objs:
+ object_name, obj = objs.popitem()
+ s3url = 's3://' + awsbucket + '/' + object_name
+ if object_name.startswith(upload_dir):
+ logging.warn(' deleting ' + s3url)
+ driver.delete_object(obj)
+ else:
+ logging.info(' skipping ' + s3url)
+def update_serverwebroot(repo_section):
+ rsyncargs = ['rsync', '-u', '-r', '--delete']
+ if options.verbose:
+ rsyncargs += ['--verbose']
+ if options.quiet:
+ rsyncargs += ['--quiet']
+ index = os.path.join(repo_section, 'index.xml')
+ indexjar = os.path.join(repo_section, 'index.jar')
+ # serverwebroot is guaranteed to have a trailing slash in common.py
+ if subprocess.call(rsyncargs +
+ ['--exclude', index, '--exclude', indexjar,
+ repo_section, config['serverwebroot']]) != 0:
+ sys.exit(1)
+ if subprocess.call(rsyncargs +
+ [index, config['serverwebroot'] + repo_section]) != 0:
+ sys.exit(1)
+ if subprocess.call(rsyncargs +
+ [indexjar, config['serverwebroot'] + repo_section]) != 0:
+ sys.exit(1)
+
+def main():
global config, options
# Parse command line...
logging.critical("The only commands currently supported are 'init' and 'update'")
sys.exit(1)
- serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/')
- host, fdroiddir = serverwebroot.split(':')
- serverrepobase = os.path.basename(fdroiddir)
if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True:
standardwebroot = False
else:
standardwebroot = True
- if serverrepobase != 'fdroid' and standardwebroot:
- print('ERROR: serverwebroot does not end with "fdroid", '
- + 'perhaps you meant one of these:\n\t'
- + serverwebroot.rstrip('/') + '/fdroid\n\t'
- + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
+
+ if 'serverwebroot' in config:
+ serverwebroot = config['serverwebroot']
+ host, fdroiddir = serverwebroot.rstrip('/').split(':')
+ serverrepobase = os.path.basename(fdroiddir)
+ if serverrepobase != 'fdroid' and standardwebroot:
+ logging.error('serverwebroot does not end with "fdroid", '
+ + 'perhaps you meant one of these:\n\t'
+ + serverwebroot.rstrip('/') + '/fdroid\n\t'
+ + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
+ sys.exit(1)
+ elif 'awsbucket' not in config:
+ logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.')
sys.exit(1)
- repodirs = ['repo']
+ repo_sections = ['repo']
if config['archive_older'] != 0:
- repodirs.append('archive')
-
- for repodir in repodirs:
- if args[0] == 'init':
- if subprocess.call(['ssh', '-v', host,
- 'mkdir -p', fdroiddir + '/' + repodir]) != 0:
- sys.exit(1)
- elif args[0] == 'update':
- index = os.path.join(repodir, 'index.xml')
- indexjar = os.path.join(repodir, 'index.jar')
- if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
- '--exclude', index, '--exclude', indexjar,
- repodir, config['serverwebroot']]) != 0:
- sys.exit(1)
- if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
- index,
- config['serverwebroot'] + '/' + repodir]) != 0:
- sys.exit(1)
- if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
- indexjar,
- config['serverwebroot'] + '/' + repodir]) != 0:
- sys.exit(1)
+ repo_sections.append('archive')
+
+ if args[0] == 'init':
+ if serverwebroot != None:
+ sshargs = ['ssh']
+ if options.quiet:
+ sshargs += ['-q']
+ for repo_section in repo_sections:
+ cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section]
+ if options.verbose:
+ # ssh -v produces different output than rsync -v, so this
+ # simulates rsync -v
+ logging.info(' '.join(cmd))
+ if subprocess.call(cmd) != 0:
+ sys.exit(1)
+ elif args[0] == 'update':
+ for repo_section in repo_sections:
+ if 'serverwebroot' in config:
+ update_serverwebroot(repo_section)
+ if 'awsbucket' in config:
+ update_awsbucket(repo_section)
sys.exit(0)
~ianmdlvl / fdroidserver.git / commitdiff