2 # -*- coding: utf-8 -*-
4 # server.py - part of the FDroid server tools
5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from optparse import OptionParser
34 def update_awsbucket(repo_section):
36 Upload the contents of the directory `repo_section` (including
37 subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
38 bucket will first be deleted.
40 Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
43 logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
44 + config['awsbucket'] + '"')
46 import libcloud.security
47 libcloud.security.VERIFY_SSL_CERT = True
48 from libcloud.storage.types import Provider, ContainerDoesNotExistError
49 from libcloud.storage.providers import get_driver
51 if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
52 logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
54 awsbucket = config['awsbucket']
56 cls = get_driver(Provider.S3)
57 driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
59 container = driver.get_container(container_name=awsbucket)
60 except ContainerDoesNotExistError:
61 container = driver.create_container(container_name=awsbucket)
62 logging.info('Created new container "' + container.name + '"')
64 upload_dir = 'fdroid/' + repo_section
66 for obj in container.list_objects():
67 if obj.name.startswith(upload_dir + '/'):
70 for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
73 file_to_upload = os.path.join(root, name)
74 object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
75 if object_name not in objs:
78 obj = objs.pop(object_name)
79 if obj.size != os.path.getsize(file_to_upload):
82 # if the sizes match, then compare by MD5
84 with open(file_to_upload, 'rb') as f:
90 if obj.hash != md5.hexdigest():
91 s3url = 's3://' + awsbucket + '/' + obj.name
92 logging.info(' deleting ' + s3url)
93 if not driver.delete_object(obj):
94 logging.warn('Could not delete ' + s3url)
98 logging.debug(' uploading "' + file_to_upload + '"...')
99 extra = {'acl': 'public-read'}
100 if file_to_upload.endswith('.sig'):
101 extra['content_type'] = 'application/pgp-signature'
102 elif file_to_upload.endswith('.asc'):
103 extra['content_type'] = 'application/pgp-signature'
104 logging.info(' uploading ' + os.path.relpath(file_to_upload)
105 + ' to s3://' + awsbucket + '/' + object_name)
106 obj = driver.upload_object(file_path=file_to_upload,
108 object_name=object_name,
111 # delete the remnants in the bucket, they do not exist locally
113 object_name, obj = objs.popitem()
114 s3url = 's3://' + awsbucket + '/' + object_name
115 if object_name.startswith(upload_dir):
116 logging.warn(' deleting ' + s3url)
117 driver.delete_object(obj)
119 logging.info(' skipping ' + s3url)
122 def update_serverwebroot(serverwebroot, repo_section):
123 rsyncargs = ['rsync', '--archive', '--delete']
125 rsyncargs += ['--verbose']
127 rsyncargs += ['--quiet']
128 if options.identity_file is not None:
129 rsyncargs += ['-e', 'ssh -i ' + options.identity_file]
130 if 'identity_file' in config:
131 rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
132 indexxml = os.path.join(repo_section, 'index.xml')
133 indexjar = os.path.join(repo_section, 'index.jar')
134 # serverwebroot is guaranteed to have a trailing slash in common.py
135 if subprocess.call(rsyncargs +
136 ['--exclude', indexxml, '--exclude', indexjar,
137 repo_section, serverwebroot]) != 0:
139 # use stricter checking on the indexes since they provide the signature
140 rsyncargs += ['--checksum']
141 sectionpath = serverwebroot + repo_section
142 if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0:
144 if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0:
148 def _local_sync(fromdir, todir):
149 rsyncargs = ['rsync', '--recursive', '--links', '--times', '--perms',
150 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
151 # use stricter rsync checking on all files since people using offline mode
152 # are already prioritizing security above ease and speed
153 rsyncargs += ['--checksum']
155 rsyncargs += ['--verbose']
157 rsyncargs += ['--quiet']
158 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
159 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
163 def sync_from_localcopy(repo_section, local_copy_dir):
164 logging.info('Syncing from local_copy_dir to this repo.')
165 # trailing slashes have a meaning in rsync which is not needed here, so
166 # make sure both paths have exactly one trailing slash
167 _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
168 repo_section.rstrip('/') + '/')
171 def update_localcopy(repo_section, local_copy_dir):
172 # local_copy_dir is guaranteed to have a trailing slash in main() below
173 _local_sync(repo_section, local_copy_dir)
177 global config, options
179 # Parse command line...
180 parser = OptionParser()
181 parser.add_option("-i", "--identity-file", default=None,
182 help="Specify an identity file to provide to SSH for rsyncing")
183 parser.add_option("--local-copy-dir", default=None,
184 help="Specify a local folder to sync the repo to")
185 parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False,
186 help="Before uploading to servers, sync from local copy dir")
187 parser.add_option("-v", "--verbose", action="store_true", default=False,
188 help="Spew out even more information than normal")
189 parser.add_option("-q", "--quiet", action="store_true", default=False,
190 help="Restrict output to warnings and errors")
191 (options, args) = parser.parse_args()
193 config = common.read_config(options)
196 logging.critical("Specify a single command")
199 if args[0] != 'init' and args[0] != 'update':
200 logging.critical("The only commands currently supported are 'init' and 'update'")
203 if config.get('nonstandardwebroot') is True:
204 standardwebroot = False
206 standardwebroot = True
208 for serverwebroot in config.get('serverwebroot', []):
209 host, fdroiddir = serverwebroot.rstrip('/').split(':')
210 repobase = os.path.basename(fdroiddir)
211 if standardwebroot and repobase != 'fdroid':
212 logging.error('serverwebroot path does not end with "fdroid", '
213 + 'perhaps you meant one of these:\n\t'
214 + serverwebroot.rstrip('/') + '/fdroid\n\t'
215 + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
218 if options.local_copy_dir is not None:
219 local_copy_dir = options.local_copy_dir
220 elif config.get('local_copy_dir'):
221 local_copy_dir = config['local_copy_dir']
223 local_copy_dir = None
224 if local_copy_dir is not None:
225 fdroiddir = local_copy_dir.rstrip('/')
226 if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
227 logging.error('local_copy_dir must be directory, not a file!')
229 if not os.path.exists(os.path.dirname(fdroiddir)):
230 logging.error('The root dir for local_copy_dir "'
231 + os.path.dirname(fdroiddir)
232 + '" does not exist!')
234 if not os.path.isabs(fdroiddir):
235 logging.error('local_copy_dir must be an absolute path!')
237 repobase = os.path.basename(fdroiddir)
238 if standardwebroot and repobase != 'fdroid':
239 logging.error('local_copy_dir does not end with "fdroid", '
240 + 'perhaps you meant: ' + fdroiddir + '/fdroid')
242 if local_copy_dir[-1] != '/':
243 local_copy_dir += '/'
244 local_copy_dir = local_copy_dir.replace('//', '/')
245 if not os.path.exists(fdroiddir):
248 if not config.get('awsbucket') \
249 and not config.get('serverwebroot') \
250 and local_copy_dir is None:
251 logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!'
252 + 'Edit your config.py to set at least one.')
255 repo_sections = ['repo']
256 if config['archive_older'] != 0:
257 repo_sections.append('archive')
258 if not os.path.exists('archive'):
261 if args[0] == 'init':
262 ssh = paramiko.SSHClient()
263 ssh.load_system_host_keys()
264 for serverwebroot in config.get('serverwebroot', []):
265 sshstr, remotepath = serverwebroot.rstrip('/').split(':')
266 if sshstr.find('@') >= 0:
267 username, hostname = sshstr.split('@')
269 username = pwd.getpwuid(os.getuid())[0] # get effective uid
271 ssh.connect(hostname, username=username)
272 sftp = ssh.open_sftp()
273 if os.path.basename(remotepath) \
274 not in sftp.listdir(os.path.dirname(remotepath)):
275 sftp.mkdir(remotepath, mode=0755)
276 for repo_section in repo_sections:
277 repo_path = os.path.join(remotepath, repo_section)
278 if os.path.basename(repo_path) \
279 not in sftp.listdir(remotepath):
280 sftp.mkdir(repo_path, mode=0755)
283 elif args[0] == 'update':
284 for repo_section in repo_sections:
285 if local_copy_dir is not None:
286 if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
287 sync_from_localcopy(repo_section, local_copy_dir)
289 update_localcopy(repo_section, local_copy_dir)
290 for serverwebroot in config.get('serverwebroot', []):
291 update_serverwebroot(serverwebroot, repo_section)
292 if config.get('awsbucket'):
293 update_awsbucket(repo_section)
297 if __name__ == "__main__":