3 # server.py - part of the FDroid server tools
4 # Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from argparse import ArgumentParser
36 def update_awsbucket(repo_section):
38 Upload the contents of the directory `repo_section` (including
39 subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
40 bucket will first be deleted.
42 Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
45 logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
46 + config['awsbucket'] + '"')
48 import libcloud.security
49 libcloud.security.VERIFY_SSL_CERT = True
50 from libcloud.storage.types import Provider, ContainerDoesNotExistError
51 from libcloud.storage.providers import get_driver
53 if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
54 logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
56 awsbucket = config['awsbucket']
58 cls = get_driver(Provider.S3)
59 driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
61 container = driver.get_container(container_name=awsbucket)
62 except ContainerDoesNotExistError:
63 container = driver.create_container(container_name=awsbucket)
64 logging.info('Created new container "' + container.name + '"')
66 upload_dir = 'fdroid/' + repo_section
68 for obj in container.list_objects():
69 if obj.name.startswith(upload_dir + '/'):
72 for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
75 file_to_upload = os.path.join(root, name)
76 object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
77 if object_name not in objs:
80 obj = objs.pop(object_name)
81 if obj.size != os.path.getsize(file_to_upload):
84 # if the sizes match, then compare by MD5
86 with open(file_to_upload, 'rb') as f:
92 if obj.hash != md5.hexdigest():
93 s3url = 's3://' + awsbucket + '/' + obj.name
94 logging.info(' deleting ' + s3url)
95 if not driver.delete_object(obj):
96 logging.warn('Could not delete ' + s3url)
100 logging.debug(' uploading "' + file_to_upload + '"...')
101 extra = {'acl': 'public-read'}
102 if file_to_upload.endswith('.sig'):
103 extra['content_type'] = 'application/pgp-signature'
104 elif file_to_upload.endswith('.asc'):
105 extra['content_type'] = 'application/pgp-signature'
106 logging.info(' uploading ' + os.path.relpath(file_to_upload)
107 + ' to s3://' + awsbucket + '/' + object_name)
108 with open(file_to_upload, 'rb') as iterator:
109 obj = driver.upload_object_via_stream(iterator=iterator,
111 object_name=object_name,
113 # delete the remnants in the bucket, they do not exist locally
115 object_name, obj = objs.popitem()
116 s3url = 's3://' + awsbucket + '/' + object_name
117 if object_name.startswith(upload_dir):
118 logging.warn(' deleting ' + s3url)
119 driver.delete_object(obj)
121 logging.info(' skipping ' + s3url)
124 def update_serverwebroot(serverwebroot, repo_section):
125 # use a checksum comparison for accurate comparisons on different
126 # filesystems, for example, FAT has a low resolution timestamp
127 rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
128 if not options.no_checksum:
129 rsyncargs.append('--checksum')
131 rsyncargs += ['--verbose']
133 rsyncargs += ['--quiet']
134 if options.identity_file is not None:
135 rsyncargs += ['-e', 'ssh -i ' + options.identity_file]
136 if 'identity_file' in config:
137 rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
138 indexxml = os.path.join(repo_section, 'index.xml')
139 indexjar = os.path.join(repo_section, 'index.jar')
140 indexv1jar = os.path.join(repo_section, 'index-v1.jar')
141 # Upload the first time without the index files and delay the deletion as
142 # much as possible, that keeps the repo functional while this update is
143 # running. Then once it is complete, rerun the command again to upload
144 # the index files. Always using the same target with rsync allows for
145 # very strict settings on the receiving server, you can literally specify
146 # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
147 # (serverwebroot is guaranteed to have a trailing slash in common.py)
148 logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
149 if subprocess.call(rsyncargs +
150 ['--exclude', indexxml, '--exclude', indexjar,
151 '--exclude', indexv1jar,
152 repo_section, serverwebroot]) != 0:
154 if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
156 # upload "current version" symlinks if requested
157 if config['make_current_version_link'] and repo_section == 'repo':
159 for f in glob.glob('*.apk') \
160 + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
161 if os.path.islink(f):
162 links_to_upload.append(f)
163 if len(links_to_upload) > 0:
164 if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
168 def _local_sync(fromdir, todir):
169 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
170 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
171 # use stricter rsync checking on all files since people using offline mode
172 # are already prioritizing security above ease and speed
173 if not options.no_checksum:
174 rsyncargs.append('--checksum')
176 rsyncargs += ['--verbose']
178 rsyncargs += ['--quiet']
179 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
180 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
184 def sync_from_localcopy(repo_section, local_copy_dir):
185 logging.info('Syncing from local_copy_dir to this repo.')
186 # trailing slashes have a meaning in rsync which is not needed here, so
187 # make sure both paths have exactly one trailing slash
188 _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
189 repo_section.rstrip('/') + '/')
192 def update_localcopy(repo_section, local_copy_dir):
193 # local_copy_dir is guaranteed to have a trailing slash in main() below
194 _local_sync(repo_section, local_copy_dir)
197 def update_servergitmirrors(servergitmirrors, repo_section):
198 # depend on GitPython only if users set a git mirror
200 # right now we support only 'repo' git-mirroring
201 if repo_section == 'repo':
202 # create a new git-mirror folder
203 repo_dir = os.path.join('.', 'git-mirror/')
205 # remove if already present
206 if os.path.isdir(repo_dir):
207 shutil.rmtree(repo_dir)
209 repo = git.Repo.init(repo_dir)
211 # take care of each mirror
212 for mirror in servergitmirrors:
213 hostname = mirror.split("/")[2]
214 repo.create_remote(hostname, mirror)
215 logging.info('Mirroring to: ' + mirror)
217 # copy local 'repo' to 'git-mirror/fdroid/repo directory' with _local_sync
218 fdroid_repo_path = os.path.join(repo_dir, "fdroid")
219 _local_sync(repo_section, fdroid_repo_path)
221 # sadly index.add don't allow the --all parameter
222 repo.git.add(all=True)
223 repo.index.commit("fdroidserver git-mirror")
225 # push for every remote. This will overwrite the git history
226 for remote in repo.remotes:
227 remote.push('master', force=True, set_upstream=True)
230 def upload_to_android_observatory(repo_section):
231 # depend on requests and lxml only if users enable AO
233 from lxml.html import fromstring
235 if repo_section == 'repo':
236 for f in glob.glob(os.path.join(repo_section, '*.apk')):
238 fname = os.path.basename(f)
239 logging.info('Uploading ' + fname + ' to androidobservatory.org')
241 # upload the file with a post request
242 r = requests.post('https://androidobservatory.org/upload', files={'apk': (fname, open(fpath, 'rb'))})
246 # from now on XPath will be used to retrieve the message in the HTML
247 # androidobservatory doesn't have a nice API to talk with
248 # so we must scrape the page content
249 tree = fromstring(response)
250 alert = tree.xpath("//html/body/div[@class='container content-container']/div[@class='alert alert-info']")[0]
255 # if the application was added successfully we retrive the url
256 # if the application was already uploaded we use the redirect page url
257 if el.attrib.get("href") is not None:
258 appurl = page + el.attrib["href"][1:]
259 message += el.text.replace(" here", "") + el.tail
262 message = message.strip() + " " + appurl
263 logging.info(message)
266 def upload_to_virustotal(repo_section, vt_apikey):
269 if repo_section == 'repo':
270 for f in glob.glob(os.path.join(repo_section, '*.apk')):
272 fname = os.path.basename(f)
273 logging.info('Uploading ' + fname + ' to virustotal.com')
275 # upload the file with a post request
276 params = {'apikey': vt_apikey}
277 files = {'file': (fname, open(fpath, 'rb'))}
278 r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=params)
281 logging.info(response['verbose_msg'] + " " + response['permalink'])
284 def push_binary_transparency(binary_transparency_remote):
285 '''push the binary transparency git repo to the specifed remote'''
288 repo = git.Repo('binary_transparency_log')
290 for remote in repo.remotes:
291 if remote.url == binary_transparency_remote:
296 pushremote = repo.create_remote('fdroid_server_update', binary_transparency_remote)
297 pushremote.push('master')
301 global config, options
303 # Parse command line...
304 parser = ArgumentParser()
305 common.setup_global_opts(parser)
306 parser.add_argument("command", help="command to execute, either 'init' or 'update'")
307 parser.add_argument("-i", "--identity-file", default=None,
308 help="Specify an identity file to provide to SSH for rsyncing")
309 parser.add_argument("--local-copy-dir", default=None,
310 help="Specify a local folder to sync the repo to")
311 parser.add_argument("--sync-from-local-copy-dir", action="store_true", default=False,
312 help="Before uploading to servers, sync from local copy dir")
313 parser.add_argument("--no-checksum", action="store_true", default=False,
314 help="Don't use rsync checksums")
315 options = parser.parse_args()
317 config = common.read_config(options)
319 if options.command != 'init' and options.command != 'update':
320 logging.critical("The only commands currently supported are 'init' and 'update'")
323 if config.get('nonstandardwebroot') is True:
324 standardwebroot = False
326 standardwebroot = True
328 for serverwebroot in config.get('serverwebroot', []):
329 # this supports both an ssh host:path and just a path
330 s = serverwebroot.rstrip('/').split(':')
336 logging.error('Malformed serverwebroot line: ' + serverwebroot)
338 repobase = os.path.basename(fdroiddir)
339 if standardwebroot and repobase != 'fdroid':
340 logging.error('serverwebroot path does not end with "fdroid", '
341 + 'perhaps you meant one of these:\n\t'
342 + serverwebroot.rstrip('/') + '/fdroid\n\t'
343 + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
346 if options.local_copy_dir is not None:
347 local_copy_dir = options.local_copy_dir
348 elif config.get('local_copy_dir'):
349 local_copy_dir = config['local_copy_dir']
351 local_copy_dir = None
352 if local_copy_dir is not None:
353 fdroiddir = local_copy_dir.rstrip('/')
354 if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
355 logging.error('local_copy_dir must be directory, not a file!')
357 if not os.path.exists(os.path.dirname(fdroiddir)):
358 logging.error('The root dir for local_copy_dir "'
359 + os.path.dirname(fdroiddir)
360 + '" does not exist!')
362 if not os.path.isabs(fdroiddir):
363 logging.error('local_copy_dir must be an absolute path!')
365 repobase = os.path.basename(fdroiddir)
366 if standardwebroot and repobase != 'fdroid':
367 logging.error('local_copy_dir does not end with "fdroid", '
368 + 'perhaps you meant: ' + fdroiddir + '/fdroid')
370 if local_copy_dir[-1] != '/':
371 local_copy_dir += '/'
372 local_copy_dir = local_copy_dir.replace('//', '/')
373 if not os.path.exists(fdroiddir):
376 if not config.get('awsbucket') \
377 and not config.get('serverwebroot') \
378 and not config.get('servergitmirrors') \
379 and not config.get('uploadto_androidobservatory') \
380 and not config.get('virustotal_apikey') \
381 and local_copy_dir is None:
382 logging.warn('No option set! Edit your config.py to set at least one among:\n'
383 + 'serverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey or uploadto_androidobservatory')
386 repo_sections = ['repo']
387 if config['archive_older'] != 0:
388 repo_sections.append('archive')
389 if not os.path.exists('archive'):
391 if config['per_app_repos']:
392 repo_sections += common.get_per_app_repos()
394 if options.command == 'init':
395 ssh = paramiko.SSHClient()
396 ssh.load_system_host_keys()
397 for serverwebroot in config.get('serverwebroot', []):
398 sshstr, remotepath = serverwebroot.rstrip('/').split(':')
399 if sshstr.find('@') >= 0:
400 username, hostname = sshstr.split('@')
402 username = pwd.getpwuid(os.getuid())[0] # get effective uid
404 ssh.connect(hostname, username=username)
405 sftp = ssh.open_sftp()
406 if os.path.basename(remotepath) \
407 not in sftp.listdir(os.path.dirname(remotepath)):
408 sftp.mkdir(remotepath, mode=0o755)
409 for repo_section in repo_sections:
410 repo_path = os.path.join(remotepath, repo_section)
411 if os.path.basename(repo_path) \
412 not in sftp.listdir(remotepath):
413 sftp.mkdir(repo_path, mode=0o755)
416 elif options.command == 'update':
417 for repo_section in repo_sections:
418 if local_copy_dir is not None:
419 if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
420 sync_from_localcopy(repo_section, local_copy_dir)
422 update_localcopy(repo_section, local_copy_dir)
423 for serverwebroot in config.get('serverwebroot', []):
424 update_serverwebroot(serverwebroot, repo_section)
425 if config.get('servergitmirrors', []):
426 # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
427 servergitmirrors = config.get('servergitmirrors', [])
428 update_servergitmirrors(servergitmirrors, repo_section)
429 if config.get('awsbucket'):
430 update_awsbucket(repo_section)
431 if config.get('uploadto_androidobservatory'):
432 upload_to_android_observatory(repo_section)
433 if config.get('virustotal_apikey'):
434 upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
436 binary_transparency_remote = config.get('binary_transparency_remote')
437 if binary_transparency_remote:
438 push_binary_transparency(binary_transparency_remote)
443 if __name__ == "__main__":