2 # -*- coding: utf-8 -*-
4 # server.py - part of the FDroid server tools
5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
24 from optparse import OptionParser
32 def update_awsbucket(repo_section):
34 Upload the contents of the directory `repo_section` (including
35 subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
36 bucket will first be deleted.
38 Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
41 import libcloud.security
42 libcloud.security.VERIFY_SSL_CERT = True
43 from libcloud.storage.types import Provider, ContainerDoesNotExistError
44 from libcloud.storage.providers import get_driver
46 if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
47 logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
49 awsbucket = config['awsbucket']
51 cls = get_driver(Provider.S3)
52 driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
54 container = driver.get_container(container_name=awsbucket)
55 except ContainerDoesNotExistError:
56 container = driver.create_container(container_name=awsbucket)
57 logging.info('Created new container "' + container.name + '"')
59 upload_dir = 'fdroid/' + repo_section
61 for obj in container.list_objects():
62 if obj.name.startswith(upload_dir + '/'):
65 for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
68 file_to_upload = os.path.join(root, name)
69 object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
70 if object_name not in objs:
73 obj = objs.pop(object_name)
74 if obj.size != os.path.getsize(file_to_upload):
77 # if the sizes match, then compare by MD5
79 with open(file_to_upload, 'rb') as f:
85 if obj.hash != md5.hexdigest():
86 s3url = 's3://' + awsbucket + '/' + obj.name
87 logging.info(' deleting ' + s3url)
88 if not driver.delete_object(obj):
89 logging.warn('Could not delete ' + s3url)
93 logging.debug(' uploading "' + file_to_upload + '"...')
94 extra = {'acl': 'public-read'}
95 if file_to_upload.endswith('.sig'):
96 extra['content_type'] = 'application/pgp-signature'
97 elif file_to_upload.endswith('.asc'):
98 extra['content_type'] = 'application/pgp-signature'
99 logging.info(' uploading ' + os.path.relpath(file_to_upload)
100 + ' to s3://' + awsbucket + '/' + object_name)
101 obj = driver.upload_object(file_path=file_to_upload,
103 object_name=object_name,
106 # delete the remnants in the bucket, they do not exist locally
108 object_name, obj = objs.popitem()
109 s3url = 's3://' + awsbucket + '/' + object_name
110 if object_name.startswith(upload_dir):
111 logging.warn(' deleting ' + s3url)
112 driver.delete_object(obj)
114 logging.info(' skipping ' + s3url)
117 def update_serverwebroot(repo_section):
118 rsyncargs = ['rsync', '--update', '--recursive', '--delete']
120 rsyncargs += ['--verbose']
122 rsyncargs += ['--quiet']
123 if options.identity_file is not None:
124 rsyncargs += ['-e', 'ssh -i ' + options.identity_file]
125 if 'identity_file' in config:
126 rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
127 indexxml = os.path.join(repo_section, 'index.xml')
128 indexjar = os.path.join(repo_section, 'index.jar')
129 # serverwebroot is guaranteed to have a trailing slash in common.py
130 if subprocess.call(rsyncargs +
131 ['--exclude', indexxml, '--exclude', indexjar,
132 repo_section, config['serverwebroot']]) != 0:
134 # use stricter checking on the indexes since they provide the signature
135 rsyncargs += ['--checksum']
136 sectionpath = config['serverwebroot'] + repo_section
137 if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0:
139 if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0:
143 def update_localcopy(repo_section, local_copy_dir):
144 rsyncargs = ['rsync', '--update', '--recursive', '--delete']
145 # use stricter rsync checking on all files since people using offline mode
146 # are already prioritizing security above ease and speed
147 rsyncargs += ['--checksum']
149 rsyncargs += ['--verbose']
151 rsyncargs += ['--quiet']
152 # local_copy_dir is guaranteed to have a trailing slash in main() below
153 if subprocess.call(rsyncargs + [repo_section, local_copy_dir]) != 0:
158 global config, options
160 # Parse command line...
161 parser = OptionParser()
162 parser.add_option("-i", "--identity-file", default=None,
163 help="Specify an identity file to provide to SSH for rsyncing")
164 parser.add_option("--local-copy-dir", default=None,
165 help="Specify a local folder to sync the repo to")
166 parser.add_option("-v", "--verbose", action="store_true", default=False,
167 help="Spew out even more information than normal")
168 parser.add_option("-q", "--quiet", action="store_true", default=False,
169 help="Restrict output to warnings and errors")
170 (options, args) = parser.parse_args()
172 config = common.read_config(options)
175 logging.critical("Specify a single command")
178 if args[0] != 'init' and args[0] != 'update':
179 logging.critical("The only commands currently supported are 'init' and 'update'")
182 if config.get('nonstandardwebroot') is True:
183 standardwebroot = False
185 standardwebroot = True
187 if config.get('serverwebroot'):
188 serverwebroot = config['serverwebroot']
189 host, fdroiddir = serverwebroot.rstrip('/').split(':')
190 repobase = os.path.basename(fdroiddir)
191 if standardwebroot and repobase != 'fdroid':
192 logging.error('serverwebroot does not end with "fdroid", '
193 + 'perhaps you meant one of these:\n\t'
194 + serverwebroot.rstrip('/') + '/fdroid\n\t'
195 + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
198 if options.local_copy_dir is not None:
199 local_copy_dir = options.local_copy_dir
200 elif config.get('local_copy_dir'):
201 local_copy_dir = config['local_copy_dir']
203 local_copy_dir = None
204 if local_copy_dir is not None:
205 fdroiddir = local_copy_dir.rstrip('/')
206 if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
207 logging.error('local_copy_dir must be directory, not a file!')
209 if not os.path.exists(os.path.dirname(fdroiddir)):
210 logging.error('The root dir for local_copy_dir "'
211 + os.path.dirname(fdroiddir)
212 + '" does not exist!')
214 if not os.path.isabs(fdroiddir):
215 logging.error('local_copy_dir must be an absolute path!')
217 repobase = os.path.basename(fdroiddir)
218 if standardwebroot and repobase != 'fdroid':
219 logging.error('local_copy_dir does not end with "fdroid", '
220 + 'perhaps you meant: ' + fdroiddir + '/fdroid')
222 if local_copy_dir[-1] != '/':
223 local_copy_dir += '/'
224 local_copy_dir = local_copy_dir.replace('//', '/')
225 if not os.path.exists(fdroiddir):
228 if not config.get('awsbucket') \
229 and not config.get('serverwebroot') \
230 and local_copy_dir is None:
231 logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!'
232 + 'Edit your config.py to set at least one.')
235 repo_sections = ['repo']
236 if config['archive_older'] != 0:
237 repo_sections.append('archive')
239 if args[0] == 'init':
240 if config.get('serverwebroot'):
244 for repo_section in repo_sections:
245 cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section]
247 # ssh -v produces different output than rsync -v, so this
249 logging.info(' '.join(cmd))
250 if subprocess.call(cmd) != 0:
252 elif args[0] == 'update':
253 for repo_section in repo_sections:
254 if config.get('serverwebroot'):
255 update_serverwebroot(repo_section)
256 if config.get('awsbucket'):
257 update_awsbucket(repo_section)
258 if local_copy_dir is not None:
259 update_localcopy(repo_section, local_copy_dir)
263 if __name__ == "__main__":