From: Daniel Martí Date: Fri, 27 Jun 2014 19:11:52 +0000 (+0200) Subject: Merge branch 'add-local_copy_dir-and-v0.2-fixes' of https://gitlab.com/eighthave... X-Git-Tag: 0.2^0 X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=df6d511a8efffe494bfe1652891ff8ca78dea6c9;hp=0ee6fc8c227221b95ce1c0a1a6a5efcd9724d75a;p=fdroidserver.git Merge branch 'add-local_copy_dir-and-v0.2-fixes' of https://gitlab.com/eighthave/fdroidserver --- diff --git a/MANIFEST.in b/MANIFEST.in index 47cd528e..93c086ab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,6 +23,10 @@ include docs/update.sh include examples/config.py include examples/fdroid-icon.png include examples/makebs.config.py +include examples/opensc-fdroid.cfg +include fdroidserver/getsig/run.sh +include fdroidserver/getsig/make.sh +include fdroidserver/getsig/getsig.java include tests/run-tests include wp-fdroid/AndroidManifest.xml include wp-fdroid/android-permissions.php diff --git a/examples/config.py b/examples/config.py index df1ec79b..221b63bf 100644 --- a/examples/config.py +++ b/examples/config.py @@ -98,28 +98,56 @@ keyaliases['com.example.app'] = 'example' # the @ prefix. keyaliases['com.example.another.plugin'] = '@com.example.another' + # The full path to the root of the repository. It must be specified in # rsync/ssh format for a remote host/path. This is used for syncing a locally # generated repo to the server that is it hosted on. It must end in the # standard public repo name of "/fdroid", but can be in up to three levels of # sub-directories (i.e. /var/www/packagerepos/fdroid). +# # serverwebroot = 'user@example:/var/www/fdroid' + # optionally specific which identity file to use when using rsync over SSH +# # identity_file = '~/.ssh/fdroid_id_rsa' + +# If you are running the repo signing process on a completely offline machine, +# which provides the best security, then you can specify a folder to sync the +# repo to when running `fdroid server update`. This is most likely going to +# be a USB thumb drive, SD Card, or some other kind of removable media. Make +# sure it is mounted before running `fdroid server update`. Using the +# standard folder called 'fdroid' as the specified folder is recommended, like +# with serverwebroot. +# +# local_copy_dir = '/media/MyUSBThumbDrive/fdroid' + + +# If you are using local_copy_dir on an offline build/signing server, once the +# thumb drive has been plugged into the online machine, it will need to be +# synced to the copy on the online machine. To make that happen +# automatically, set sync_from_local_copy_dir to True: +# +# sync_from_local_copy_dir = True + + # To upload the repo to an Amazon S3 bucket using `fdroid server update`. # Warning, this deletes and recreates the whole fdroid/ directory each # time. This is based on apache-libcloud, which supports basically all cloud # storage services, so it should be easy to port the fdroid server tools to # any of them. +# # awsbucket = 'myawsfdroid' # awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA' # awssecretkey = 'yourverysecretkeywordpassphraserighthere' + # If you want to force 'fdroid server' to use a non-standard serverwebroot +# # nonstandardwebroot = False + # Wiki details wiki_protocol = "http" wiki_server = "server" diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a4b3fb0f..afafffdf 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -47,6 +47,7 @@ def get_default_config(): 'mvn3': "mvn", 'gradle': 'gradle', 'archive_older': 0, + 'sync_from_local_copy_dir': False, 'update_stats': False, 'stats_to_carbon': False, 'repo_maxage': 0, diff --git a/fdroidserver/server.py b/fdroidserver/server.py index bddc4b0f..9b46de70 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -115,7 +115,7 @@ def update_awsbucket(repo_section): def update_serverwebroot(repo_section): - rsyncargs = ['rsync', '--update', '--recursive', '--delete'] + rsyncargs = ['rsync', '--archive', '--delete'] if options.verbose: rsyncargs += ['--verbose'] if options.quiet: @@ -140,6 +140,33 @@ def update_serverwebroot(repo_section): sys.exit(1) +def _local_sync(fromdir, todir): + rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete'] + # use stricter rsync checking on all files since people using offline mode + # are already prioritizing security above ease and speed + rsyncargs += ['--checksum'] + if options.verbose: + rsyncargs += ['--verbose'] + if options.quiet: + rsyncargs += ['--quiet'] + logging.debug(' '.join(rsyncargs + [fromdir, todir])) + if subprocess.call(rsyncargs + [fromdir, todir]) != 0: + sys.exit(1) + + +def sync_from_localcopy(repo_section, local_copy_dir): + logging.info('Syncing from local_copy_dir to this repo.') + # trailing slashes have a meaning in rsync which is not needed here, so + # remove them all + _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/'), + repo_section.rstrip('/')) + + +def update_localcopy(repo_section, local_copy_dir): + # local_copy_dir is guaranteed to have a trailing slash in main() below + _local_sync(repo_section, local_copy_dir) + + def main(): global config, options @@ -147,6 +174,10 @@ def main(): parser = OptionParser() parser.add_option("-i", "--identity-file", default=None, help="Specify an identity file to provide to SSH for rsyncing") + parser.add_option("--local-copy-dir", default=None, + help="Specify a local folder to sync the repo to") + parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False, + help="Before uploading to servers, sync from local copy dir") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, @@ -171,20 +202,56 @@ def main(): if config.get('serverwebroot'): serverwebroot = config['serverwebroot'] host, fdroiddir = serverwebroot.rstrip('/').split(':') - serverrepobase = os.path.basename(fdroiddir) - if serverrepobase != 'fdroid' and standardwebroot: + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': logging.error('serverwebroot does not end with "fdroid", ' + 'perhaps you meant one of these:\n\t' + serverwebroot.rstrip('/') + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid') + + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') + sys.exit(1) + + if options.local_copy_dir is not None: + local_copy_dir = options.local_copy_dir + elif config.get('local_copy_dir'): + local_copy_dir = config['local_copy_dir'] + else: + local_copy_dir = None + if local_copy_dir is not None: + fdroiddir = local_copy_dir.rstrip('/') + if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir): + logging.error('local_copy_dir must be directory, not a file!') sys.exit(1) - elif not config.get('awsbucket'): - logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.') + if not os.path.exists(os.path.dirname(fdroiddir)): + logging.error('The root dir for local_copy_dir "' + + os.path.dirname(fdroiddir) + + '" does not exist!') + sys.exit(1) + if not os.path.isabs(fdroiddir): + logging.error('local_copy_dir must be an absolute path!') + sys.exit(1) + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': + logging.error('local_copy_dir does not end with "fdroid", ' + + 'perhaps you meant: ' + fdroiddir + '/fdroid') + sys.exit(1) + if local_copy_dir[-1] != '/': + local_copy_dir += '/' + local_copy_dir = local_copy_dir.replace('//', '/') + if not os.path.exists(fdroiddir): + os.mkdir(fdroiddir) + + if not config.get('awsbucket') \ + and not config.get('serverwebroot') \ + and local_copy_dir is None: + logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!' + + 'Edit your config.py to set at least one.') sys.exit(1) repo_sections = ['repo'] if config['archive_older'] != 0: repo_sections.append('archive') + if not os.path.exists('archive'): + os.mkdir('archive') if args[0] == 'init': if config.get('serverwebroot'): @@ -201,6 +268,11 @@ def main(): sys.exit(1) elif args[0] == 'update': for repo_section in repo_sections: + if local_copy_dir is not None: + if config['sync_from_local_copy_dir'] and os.path.exists(repo_section): + sync_from_localcopy(repo_section, local_copy_dir) + else: + update_localcopy(repo_section, local_copy_dir) if config.get('serverwebroot'): update_serverwebroot(repo_section) if config.get('awsbucket'): diff --git a/setup.py b/setup.py index a3fcdcc8..d4566f07 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ if not os.path.exists('fdroidserver/getsig/getsig.class'): shell=True) setup(name='fdroidserver', - version='0.1', + version='0.2', description='F-Droid Server Tools', long_description=open('README').read(), author='The F-Droid Project', diff --git a/tests/run-tests b/tests/run-tests index 72b8b3f3..a5576765 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -71,7 +71,42 @@ if [ -z $aapt ]; then fi #------------------------------------------------------------------------------# -echo_header "setup a new repo from scratch using ANDROID_HOME" +echo_header "test config checks of local_copy_dir" + +REPOROOT=`create_test_dir` +cd $REPOROOT +$fdroid init +$fdroid update --create-metadata +$fdroid server update --local-copy-dir=/tmp/fdroid + +# now test the errors work +set +e +$fdroid server update --local-copy-dir=thisisnotanabsolutepath +if [ $? -eq 0 ]; then + echo "This should have failed because thisisnotanabsolutepath is not an absolute path!" + exit 1 +else + echo "testing absolute path checker passed" +fi +$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf +if [ $? -eq 0 ]; then + echo "This should have failed because the path does not end with 'fdroid'!" + exit 1 +else + echo "testing dirname exists checker passed" +fi +$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf/fdroid +if [ $? -eq 0 ]; then + echo "This should have failed because the dirname path does not exist!" + exit 1 +else + echo "testing dirname exists checker passed" +fi +set -e + + +#------------------------------------------------------------------------------# +echo_header "setup a new repo from scratch using ANDROID_HOME and do a local sync" REPOROOT=`create_test_dir` cd $REPOROOT @@ -80,6 +115,13 @@ copy_apks_into_repo $REPOROOT $fdroid update --create-metadata grep -F '