-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# server.py - part of the FDroid server tools
# Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
import paramiko
import pwd
import subprocess
-from optparse import OptionParser
+from argparse import ArgumentParser
import logging
import common
extra['content_type'] = 'application/pgp-signature'
logging.info(' uploading ' + os.path.relpath(file_to_upload)
+ ' to s3://' + awsbucket + '/' + object_name)
- obj = driver.upload_object(file_path=file_to_upload,
- container=container,
- object_name=object_name,
- verify_hash=False,
- extra=extra)
+ with open(file_to_upload, 'rb') as iterator:
+ obj = driver.upload_object_via_stream(iterator=iterator,
+ container=container,
+ object_name=object_name,
+ extra=extra)
# delete the remnants in the bucket, they do not exist locally
while objs:
object_name, obj = objs.popitem()
def update_serverwebroot(serverwebroot, repo_section):
# use a checksum comparison for accurate comparisons on different
# filesystems, for example, FAT has a low resolution timestamp
- rsyncargs = ['rsync', '--archive', '--delete']
+ rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
if not options.no_checksum:
rsyncargs.append('--checksum')
if options.verbose:
rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
indexxml = os.path.join(repo_section, 'index.xml')
indexjar = os.path.join(repo_section, 'index.jar')
- # upload the first time without the index so that the repo stays working
- # while this update is running. Then once it is complete, rerun the
- # command again to upload the index. Always using the same target with
- # rsync allows for very strict settings on the receiving server, you can
- # literally specify the one rsync command that is allowed to run in
- # ~/.ssh/authorized_keys. (serverwebroot is guaranteed to have a trailing
- # slash in common.py)
+ # Upload the first time without the index files and delay the deletion as
+ # much as possible, that keeps the repo functional while this update is
+ # running. Then once it is complete, rerun the command again to upload
+ # the index files. Always using the same target with rsync allows for
+ # very strict settings on the receiving server, you can literally specify
+ # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
+ # (serverwebroot is guaranteed to have a trailing slash in common.py)
+ logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
if subprocess.call(rsyncargs +
['--exclude', indexxml, '--exclude', indexjar,
repo_section, serverwebroot]) != 0:
def _local_sync(fromdir, todir):
- rsyncargs = ['rsync', '--recursive', '--links', '--times', '--perms',
+ rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
'--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
# use stricter rsync checking on all files since people using offline mode
# are already prioritizing security above ease and speed
global config, options
# Parse command line...
- parser = OptionParser()
- parser.add_option("-i", "--identity-file", default=None,
- help="Specify an identity file to provide to SSH for rsyncing")
- parser.add_option("--local-copy-dir", default=None,
- help="Specify a local folder to sync the repo to")
- parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False,
- help="Before uploading to servers, sync from local copy dir")
- parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
- parser.add_option("-q", "--quiet", action="store_true", default=False,
- help="Restrict output to warnings and errors")
- parser.add_option("--no-checksum", action="store_true", default=False,
- help="Don't use rsync checksums")
- (options, args) = parser.parse_args()
+ parser = ArgumentParser()
+ common.setup_global_opts(parser)
+ parser.add_argument("command", help="command to execute, either 'init' or 'update'")
+ parser.add_argument("-i", "--identity-file", default=None,
+ help="Specify an identity file to provide to SSH for rsyncing")
+ parser.add_argument("--local-copy-dir", default=None,
+ help="Specify a local folder to sync the repo to")
+ parser.add_argument("--sync-from-local-copy-dir", action="store_true", default=False,
+ help="Before uploading to servers, sync from local copy dir")
+ parser.add_argument("--no-checksum", action="store_true", default=False,
+ help="Don't use rsync checksums")
+ options = parser.parse_args()
config = common.read_config(options)
- if len(args) != 1:
- logging.critical("Specify a single command")
- sys.exit(1)
-
- if args[0] != 'init' and args[0] != 'update':
+ if options.command != 'init' and options.command != 'update':
logging.critical("The only commands currently supported are 'init' and 'update'")
sys.exit(1)
standardwebroot = True
for serverwebroot in config.get('serverwebroot', []):
- host, fdroiddir = serverwebroot.rstrip('/').split(':')
+ # this supports both an ssh host:path and just a path
+ s = serverwebroot.rstrip('/').split(':')
+ if len(s) == 1:
+ fdroiddir = s[0]
+ elif len(s) == 2:
+ host, fdroiddir = s
+ else:
+ logging.error('Malformed serverwebroot line: ' + serverwebroot)
+ sys.exit(1)
repobase = os.path.basename(fdroiddir)
if standardwebroot and repobase != 'fdroid':
logging.error('serverwebroot path does not end with "fdroid", '
repo_sections.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
+ if config['per_app_repos']:
+ repo_sections += common.get_per_app_repos()
- if args[0] == 'init':
+ if options.command == 'init':
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
for serverwebroot in config.get('serverwebroot', []):
sftp.mkdir(repo_path, mode=0755)
sftp.close()
ssh.close()
- elif args[0] == 'update':
+ elif options.command == 'update':
for repo_section in repo_sections:
if local_copy_dir is not None:
if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
~ianmdlvl / fdroidserver.git / blobdiff