fdroidserver/server.py

   1 #!/usr/bin/env python3
   2 #
   3 # server.py - part of the FDroid server tools
   4 # Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU Affero General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU Affero General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Affero General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import glob
  21 import hashlib
  22 import os
  23 import paramiko
  24 import pwd
  25 import re
  26 import subprocess
  27 import time
  28 from argparse import ArgumentParser
  29 import logging
  30 import shutil
  31
  32 from . import _
  33 from . import common
  34 from .exception import FDroidException
  35
  36 config = None
  37 options = None
  38
  39 BINARY_TRANSPARENCY_DIR = 'binary_transparency'
  40
  41
  42 def update_awsbucket(repo_section):
  43     '''
  44     Upload the contents of the directory `repo_section` (including
  45     subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
  46     bucket will first be deleted.
  47
  48     Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
  49     '''
  50
  51     logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
  52                   + config['awsbucket'] + '"')
  53
  54     if common.set_command_in_config('s3cmd'):
  55         update_awsbucket_s3cmd(repo_section)
  56     else:
  57         update_awsbucket_libcloud(repo_section)
  58
  59
  60 def update_awsbucket_s3cmd(repo_section):
  61     '''upload using the CLI tool s3cmd, which provides rsync-like sync
  62
  63     The upload is done in multiple passes to reduce the chance of
  64     interfering with an existing client-server interaction.  In the
  65     first pass, only new files are uploaded.  In the second pass,
  66     changed files are uploaded, overwriting what is on the server.  On
  67     the third/last pass, the indexes are uploaded, and any removed
  68     files are deleted from the server.  The last pass is the only pass
  69     to use a full MD5 checksum of all files to detect changes.
  70     '''
  71
  72     logging.debug(_('Using s3cmd to sync with: {url}')
  73                   .format(url=config['awsbucket']))
  74
  75     configfilename = '.s3cfg'
  76     fd = os.open(configfilename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
  77     os.write(fd, '[default]\n'.encode('utf-8'))
  78     os.write(fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8'))
  79     os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8'))
  80     os.close(fd)
  81
  82     s3bucketurl = 's3://' + config['awsbucket']
  83     s3cmd = [config['s3cmd'], '--config=' + configfilename]
  84     if subprocess.call(s3cmd + ['info', s3bucketurl]) != 0:
  85         logging.warning(_('Creating new S3 bucket: {url}')
  86                         .format(url=s3bucketurl))
  87         if subprocess.call(s3cmd + ['mb', s3bucketurl]) != 0:
  88             logging.error(_('Failed to create S3 bucket: {url}')
  89                           .format(url=s3bucketurl))
  90             raise FDroidException()
  91
  92     s3cmd_sync = s3cmd + ['sync', '--acl-public']
  93     if options.verbose:
  94         s3cmd_sync += ['--verbose']
  95     if options.quiet:
  96         s3cmd_sync += ['--quiet']
  97     indexxml = os.path.join(repo_section, 'index.xml')
  98     indexjar = os.path.join(repo_section, 'index.jar')
  99     indexv1jar = os.path.join(repo_section, 'index-v1.jar')
 100
 101     s3url = s3bucketurl + '/fdroid/'
 102     logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
 103     logging.debug(_('Running first pass with MD5 checking disabled'))
 104     if subprocess.call(s3cmd_sync +
 105                        ['--no-check-md5', '--skip-existing',
 106                         '--exclude', indexxml,
 107                         '--exclude', indexjar,
 108                         '--exclude', indexv1jar,
 109                         repo_section, s3url]) != 0:
 110         raise FDroidException()
 111     logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
 112     if subprocess.call(s3cmd_sync +
 113                        ['--no-check-md5',
 114                         '--exclude', indexxml,
 115                         '--exclude', indexjar,
 116                         '--exclude', indexv1jar,
 117                         repo_section, s3url]) != 0:
 118         raise FDroidException()
 119
 120     logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
 121                   .format(path=repo_section, url=s3url))
 122     s3cmd_sync.append('--delete-removed')
 123     s3cmd_sync.append('--delete-after')
 124     if options.no_checksum:
 125         s3cmd_sync.append('--no-check-md5')
 126     else:
 127         s3cmd_sync.append('--check-md5')
 128     if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
 129         raise FDroidException()
 130
 131
 132 def update_awsbucket_libcloud(repo_section):
 133     '''
 134     Upload the contents of the directory `repo_section` (including
 135     subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
 136     bucket will first be deleted.
 137
 138     Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
 139     '''
 140
 141     logging.debug(_('using Apache libcloud to sync with {url}')
 142                   .format(url=config['awsbucket']))
 143
 144     import libcloud.security
 145     libcloud.security.VERIFY_SSL_CERT = True
 146     from libcloud.storage.types import Provider, ContainerDoesNotExistError
 147     from libcloud.storage.providers import get_driver
 148
 149     if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
 150         raise FDroidException(
 151             _('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.py!'))
 152     awsbucket = config['awsbucket']
 153
 154     cls = get_driver(Provider.S3)
 155     driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
 156     try:
 157         container = driver.get_container(container_name=awsbucket)
 158     except ContainerDoesNotExistError:
 159         container = driver.create_container(container_name=awsbucket)
 160         logging.info(_('Created new container "{name}"')
 161                      .format(name=container.name))
 162
 163     upload_dir = 'fdroid/' + repo_section
 164     objs = dict()
 165     for obj in container.list_objects():
 166         if obj.name.startswith(upload_dir + '/'):
 167             objs[obj.name] = obj
 168
 169     for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
 170         for name in files:
 171             upload = False
 172             file_to_upload = os.path.join(root, name)
 173             object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
 174             if object_name not in objs:
 175                 upload = True
 176             else:
 177                 obj = objs.pop(object_name)
 178                 if obj.size != os.path.getsize(file_to_upload):
 179                     upload = True
 180                 else:
 181                     # if the sizes match, then compare by MD5
 182                     md5 = hashlib.md5()
 183                     with open(file_to_upload, 'rb') as f:
 184                         while True:
 185                             data = f.read(8192)
 186                             if not data:
 187                                 break
 188                             md5.update(data)
 189                     if obj.hash != md5.hexdigest():
 190                         s3url = 's3://' + awsbucket + '/' + obj.name
 191                         logging.info(' deleting ' + s3url)
 192                         if not driver.delete_object(obj):
 193                             logging.warn('Could not delete ' + s3url)
 194                         upload = True
 195
 196             if upload:
 197                 logging.debug(' uploading "' + file_to_upload + '"...')
 198                 extra = {'acl': 'public-read'}
 199                 if file_to_upload.endswith('.sig'):
 200                     extra['content_type'] = 'application/pgp-signature'
 201                 elif file_to_upload.endswith('.asc'):
 202                     extra['content_type'] = 'application/pgp-signature'
 203                 logging.info(' uploading ' + os.path.relpath(file_to_upload)
 204                              + ' to s3://' + awsbucket + '/' + object_name)
 205                 with open(file_to_upload, 'rb') as iterator:
 206                     obj = driver.upload_object_via_stream(iterator=iterator,
 207                                                           container=container,
 208                                                           object_name=object_name,
 209                                                           extra=extra)
 210     # delete the remnants in the bucket, they do not exist locally
 211     while objs:
 212         object_name, obj = objs.popitem()
 213         s3url = 's3://' + awsbucket + '/' + object_name
 214         if object_name.startswith(upload_dir):
 215             logging.warn(' deleting ' + s3url)
 216             driver.delete_object(obj)
 217         else:
 218             logging.info(' skipping ' + s3url)
 219
 220
 221 def update_serverwebroot(serverwebroot, repo_section):
 222     # use a checksum comparison for accurate comparisons on different
 223     # filesystems, for example, FAT has a low resolution timestamp
 224     rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
 225     if not options.no_checksum:
 226         rsyncargs.append('--checksum')
 227     if options.verbose:
 228         rsyncargs += ['--verbose']
 229     if options.quiet:
 230         rsyncargs += ['--quiet']
 231     if options.identity_file is not None:
 232         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file]
 233     elif 'identity_file' in config:
 234         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
 235     indexxml = os.path.join(repo_section, 'index.xml')
 236     indexjar = os.path.join(repo_section, 'index.jar')
 237     indexv1jar = os.path.join(repo_section, 'index-v1.jar')
 238     # Upload the first time without the index files and delay the deletion as
 239     # much as possible, that keeps the repo functional while this update is
 240     # running.  Then once it is complete, rerun the command again to upload
 241     # the index files.  Always using the same target with rsync allows for
 242     # very strict settings on the receiving server, you can literally specify
 243     # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
 244     # (serverwebroot is guaranteed to have a trailing slash in common.py)
 245     logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
 246     if subprocess.call(rsyncargs +
 247                        ['--exclude', indexxml, '--exclude', indexjar,
 248                         '--exclude', indexv1jar,
 249                         repo_section, serverwebroot]) != 0:
 250         raise FDroidException()
 251     if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
 252         raise FDroidException()
 253     # upload "current version" symlinks if requested
 254     if config['make_current_version_link'] and repo_section == 'repo':
 255         links_to_upload = []
 256         for f in glob.glob('*.apk') \
 257                 + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
 258             if os.path.islink(f):
 259                 links_to_upload.append(f)
 260         if len(links_to_upload) > 0:
 261             if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
 262                 raise FDroidException()
 263
 264
 265 def _local_sync(fromdir, todir):
 266     rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
 267                  '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
 268     # use stricter rsync checking on all files since people using offline mode
 269     # are already prioritizing security above ease and speed
 270     if not options.no_checksum:
 271         rsyncargs.append('--checksum')
 272     if options.verbose:
 273         rsyncargs += ['--verbose']
 274     if options.quiet:
 275         rsyncargs += ['--quiet']
 276     logging.debug(' '.join(rsyncargs + [fromdir, todir]))
 277     if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
 278         raise FDroidException()
 279
 280
 281 def sync_from_localcopy(repo_section, local_copy_dir):
 282     '''Syncs the repo from "local copy dir" filesystem to this box
 283
 284     In setups that use offline signing, this is the last step that
 285     syncs the repo from the "local copy dir" e.g. a thumb drive to the
 286     repo on the local filesystem.  That local repo is then used to
 287     push to all the servers that are configured.
 288
 289     '''
 290     logging.info('Syncing from local_copy_dir to this repo.')
 291     # trailing slashes have a meaning in rsync which is not needed here, so
 292     # make sure both paths have exactly one trailing slash
 293     _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
 294                 repo_section.rstrip('/') + '/')
 295
 296     offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
 297     if os.path.exists(os.path.join(offline_copy, '.git')):
 298         online_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
 299         push_binary_transparency(offline_copy, online_copy)
 300
 301
 302 def update_localcopy(repo_section, local_copy_dir):
 303     '''copy data from offline to the "local copy dir" filesystem
 304
 305     This updates the copy of this repo used to shuttle data from an
 306     offline signing machine to the online machine, e.g. on a thumb
 307     drive.
 308
 309     '''
 310     # local_copy_dir is guaranteed to have a trailing slash in main() below
 311     _local_sync(repo_section, local_copy_dir)
 312
 313     offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
 314     if os.path.isdir(os.path.join(offline_copy, '.git')):
 315         online_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
 316         push_binary_transparency(offline_copy, online_copy)
 317
 318
 319 def _get_size(start_path='.'):
 320     '''get size of all files in a dir https://stackoverflow.com/a/1392549'''
 321     total_size = 0
 322     for root, dirs, files in os.walk(start_path):
 323         for f in files:
 324             fp = os.path.join(root, f)
 325             total_size += os.path.getsize(fp)
 326     return total_size
 327
 328
 329 def update_servergitmirrors(servergitmirrors, repo_section):
 330     '''update repo mirrors stored in git repos
 331
 332     This is a hack to use public git repos as F-Droid repos.  It
 333     recreates the git repo from scratch each time, so that there is no
 334     history.  That keeps the size of the git repo small.  Services
 335     like GitHub or GitLab have a size limit of something like 1 gig.
 336     This git repo is only a git repo for the purpose of being hosted.
 337     For history, there is the archive section, and there is the binary
 338     transparency log.
 339
 340     '''
 341     import git
 342     from clint.textui import progress
 343     if config.get('local_copy_dir') \
 344        and not config.get('sync_from_local_copy_dir'):
 345         logging.debug('Offline machine, skipping git mirror generation until `fdroid server update`')
 346         return
 347
 348     # right now we support only 'repo' git-mirroring
 349     if repo_section == 'repo':
 350         git_mirror_path = 'git-mirror'
 351         dotgit = os.path.join(git_mirror_path, '.git')
 352         git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section)
 353         if not os.path.isdir(git_repodir):
 354             os.makedirs(git_repodir)
 355         if os.path.isdir(dotgit) and _get_size(git_mirror_path) > 1000000000:
 356             logging.warning('Deleting git-mirror history, repo is too big (1 gig max)')
 357             shutil.rmtree(dotgit)
 358
 359         # rsync is very particular about trailing slashes
 360         _local_sync(repo_section.rstrip('/') + '/', git_repodir.rstrip('/') + '/')
 361
 362         # use custom SSH command if identity_file specified
 363         ssh_cmd = 'ssh -oBatchMode=yes'
 364         if options.identity_file is not None:
 365             ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % options.identity_file
 366         elif 'identity_file' in config:
 367             ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
 368
 369         repo = git.Repo.init(git_mirror_path)
 370
 371         for remote_url in servergitmirrors:
 372             hostname = re.sub(r'\W*\w+\W+(\w+).*', r'\1', remote_url)
 373             r = git.remote.Remote(repo, hostname)
 374             if r in repo.remotes:
 375                 r = repo.remote(hostname)
 376                 if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
 377                     r.set_url(remote_url)
 378             else:
 379                 repo.create_remote(hostname, remote_url)
 380             logging.info('Mirroring to: ' + remote_url)
 381
 382         # sadly index.add don't allow the --all parameter
 383         logging.debug('Adding all files to git mirror')
 384         repo.git.add(all=True)
 385         logging.debug('Committing all files into git mirror')
 386         repo.index.commit("fdroidserver git-mirror")
 387
 388         if options.verbose:
 389             bar = progress.Bar()
 390
 391             class MyProgressPrinter(git.RemoteProgress):
 392                 def update(self, op_code, current, maximum=None, message=None):
 393                     if isinstance(maximum, float):
 394                         bar.show(current, maximum)
 395             progress = MyProgressPrinter()
 396         else:
 397             progress = None
 398
 399         # push for every remote. This will overwrite the git history
 400         for remote in repo.remotes:
 401             if remote.name == 'gitlab':
 402                 logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
 403                 with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as out_file:
 404                     out_file.write("""pages:
 405   script:
 406    - mkdir .public
 407    - cp -r * .public/
 408    - mv .public public
 409   artifacts:
 410     paths:
 411     - public
 412 """)
 413
 414                 repo.git.add(all=True)
 415                 repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 416
 417             logging.debug(_('Pushing to {url}').format(url=remote.url))
 418             with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
 419                 pushinfos = remote.push('master', force=True, set_upstream=True, progress=progress)
 420                 for pushinfo in pushinfos:
 421                     if pushinfo.flags & (git.remote.PushInfo.ERROR
 422                                          | git.remote.PushInfo.REJECTED
 423                                          | git.remote.PushInfo.REMOTE_FAILURE
 424                                          | git.remote.PushInfo.REMOTE_REJECTED):
 425                         raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
 426                                               + ' ' + pushinfo.summary)
 427                     else:
 428                         logging.debug(remote.url + ': ' + pushinfo.summary)
 429
 430         if progress:
 431             bar.done()
 432
 433
 434 def upload_to_android_observatory(repo_section):
 435     # depend on requests and lxml only if users enable AO
 436     import requests
 437     from lxml.html import fromstring
 438
 439     if repo_section == 'repo':
 440         for f in glob.glob(os.path.join(repo_section, '*.apk')):
 441             fpath = f
 442             fname = os.path.basename(f)
 443             logging.info('Uploading ' + fname + ' to androidobservatory.org')
 444
 445             # upload the file with a post request
 446             r = requests.post('https://androidobservatory.org/upload', files={'apk': (fname, open(fpath, 'rb'))})
 447             response = r.text
 448             page = r.url
 449
 450             # from now on XPath will be used to retrieve the message in the HTML
 451             # androidobservatory doesn't have a nice API to talk with
 452             # so we must scrape the page content
 453             tree = fromstring(response)
 454             alert = tree.xpath("//html/body/div[@class='container content-container']/div[@class='alert alert-info']")[0]
 455
 456             message = ""
 457             appurl = page
 458             for el in alert:
 459                 # if the application was added successfully we retrive the url
 460                 # if the application was already uploaded we use the redirect page url
 461                 if el.attrib.get("href") is not None:
 462                     appurl = page + el.attrib["href"][1:]
 463                     message += el.text.replace(" here", "") + el.tail
 464                 else:
 465                     message += el.tail
 466             message = message.strip() + " " + appurl
 467             logging.info(message)
 468
 469
 470 def upload_to_virustotal(repo_section, vt_apikey):
 471     import json
 472     import requests
 473
 474     logging.getLogger("urllib3").setLevel(logging.WARNING)
 475     logging.getLogger("requests").setLevel(logging.WARNING)
 476
 477     if repo_section == 'repo':
 478         if not os.path.exists('virustotal'):
 479             os.mkdir('virustotal')
 480         with open(os.path.join(repo_section, 'index-v1.json')) as fp:
 481             index = json.load(fp)
 482         for packageName, packages in index['packages'].items():
 483             for package in packages:
 484                 outputfilename = os.path.join('virustotal',
 485                                               packageName + '_' + str(package.get('versionCode'))
 486                                               + '_' + package['hash'] + '.json')
 487                 if os.path.exists(outputfilename):
 488                     logging.debug(package['apkName'] + ' results are in ' + outputfilename)
 489                     continue
 490                 filename = package['apkName']
 491                 repofilename = os.path.join(repo_section, filename)
 492                 logging.info('Checking if ' + repofilename + ' is on virustotal')
 493
 494                 headers = {
 495                     "User-Agent": "F-Droid"
 496                 }
 497                 params = {
 498                     'apikey': vt_apikey,
 499                     'resource': package['hash'],
 500                 }
 501                 needs_file_upload = False
 502                 while True:
 503                     r = requests.post('https://www.virustotal.com/vtapi/v2/file/report',
 504                                       params=params, headers=headers)
 505                     if r.status_code == 200:
 506                         response = r.json()
 507                         if response['response_code'] == 0:
 508                             needs_file_upload = True
 509                         else:
 510                             response['filename'] = filename
 511                             response['packageName'] = packageName
 512                             response['versionCode'] = package.get('versionCode')
 513                             response['versionName'] = package.get('versionName')
 514                             with open(outputfilename, 'w') as fp:
 515                                 json.dump(response, fp, indent=2, sort_keys=True)
 516
 517                         if response.get('positives') > 0:
 518                             logging.warning(repofilename + ' has been flagged by virustotal '
 519                                             + str(response['positives']) + ' times:'
 520                                             + '\n\t' + response['permalink'])
 521                         break
 522                     elif r.status_code == 204:
 523                         time.sleep(10)  # wait for public API rate limiting
 524
 525                 if needs_file_upload:
 526                     logging.info('Uploading ' + repofilename + ' to virustotal')
 527                     files = {
 528                         'file': (filename, open(repofilename, 'rb'))
 529                     }
 530                     r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan',
 531                                       params=params, headers=headers, files=files)
 532                     response = r.json()
 533
 534                     logging.info(response['verbose_msg'] + " " + response['permalink'])
 535
 536
 537 def push_binary_transparency(git_repo_path, git_remote):
 538     '''push the binary transparency git repo to the specifed remote.
 539
 540     If the remote is a local directory, make sure it exists, and is a
 541     git repo.  This is used to move this git repo from an offline
 542     machine onto a flash drive, then onto the online machine. Also,
 543     this pulls because pushing to a non-bare git repo is error prone.
 544
 545     This is also used in offline signing setups, where it then also
 546     creates a "local copy dir" git repo that serves to shuttle the git
 547     data from the offline machine to the online machine.  In that
 548     case, git_remote is a dir on the local file system, e.g. a thumb
 549     drive.
 550
 551     '''
 552     import git
 553
 554     logging.info(_('Pushing binary transparency log to {url}')
 555                  .format(url=git_remote))
 556
 557     if os.path.isdir(os.path.dirname(git_remote)):
 558         # from offline machine to thumbdrive
 559         remote_path = os.path.abspath(git_repo_path)
 560         if not os.path.isdir(os.path.join(git_remote, '.git')):
 561             os.makedirs(git_remote, exist_ok=True)
 562             thumbdriverepo = git.Repo.init(git_remote)
 563             local = thumbdriverepo.create_remote('local', remote_path)
 564         else:
 565             thumbdriverepo = git.Repo(git_remote)
 566             local = git.remote.Remote(thumbdriverepo, 'local')
 567             if local in thumbdriverepo.remotes:
 568                 local = thumbdriverepo.remote('local')
 569                 if 'set_url' in dir(local):  # force remote URL if using GitPython 2.x
 570                     local.set_url(remote_path)
 571             else:
 572                 local = thumbdriverepo.create_remote('local', remote_path)
 573         local.pull('master')
 574     else:
 575         # from online machine to remote on a server on the internet
 576         gitrepo = git.Repo(git_repo_path)
 577         origin = git.remote.Remote(gitrepo, 'origin')
 578         if origin in gitrepo.remotes:
 579             origin = gitrepo.remote('origin')
 580             if 'set_url' in dir(origin):  # added in GitPython 2.x
 581                 origin.set_url(git_remote)
 582         else:
 583             origin = gitrepo.create_remote('origin', git_remote)
 584         origin.push('master')
 585
 586
 587 def main():
 588     global config, options
 589
 590     # Parse command line...
 591     parser = ArgumentParser()
 592     common.setup_global_opts(parser)
 593     parser.add_argument("command", help=_("command to execute, either 'init' or 'update'"))
 594     parser.add_argument("-i", "--identity-file", default=None,
 595                         help=_("Specify an identity file to provide to SSH for rsyncing"))
 596     parser.add_argument("--local-copy-dir", default=None,
 597                         help=_("Specify a local folder to sync the repo to"))
 598     parser.add_argument("--no-checksum", action="store_true", default=False,
 599                         help=_("Don't use rsync checksums"))
 600     options = parser.parse_args()
 601
 602     config = common.read_config(options)
 603
 604     if options.command != 'init' and options.command != 'update':
 605         logging.critical(_("The only commands currently supported are 'init' and 'update'"))
 606         sys.exit(1)
 607
 608     if config.get('nonstandardwebroot') is True:
 609         standardwebroot = False
 610     else:
 611         standardwebroot = True
 612
 613     for serverwebroot in config.get('serverwebroot', []):
 614         # this supports both an ssh host:path and just a path
 615         s = serverwebroot.rstrip('/').split(':')
 616         if len(s) == 1:
 617             fdroiddir = s[0]
 618         elif len(s) == 2:
 619             host, fdroiddir = s
 620         else:
 621             logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
 622             sys.exit(1)
 623         repobase = os.path.basename(fdroiddir)
 624         if standardwebroot and repobase != 'fdroid':
 625             logging.error('serverwebroot path does not end with "fdroid", '
 626                           + 'perhaps you meant one of these:\n\t'
 627                           + serverwebroot.rstrip('/') + '/fdroid\n\t'
 628                           + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
 629             sys.exit(1)
 630
 631     if options.local_copy_dir is not None:
 632         local_copy_dir = options.local_copy_dir
 633     elif config.get('local_copy_dir'):
 634         local_copy_dir = config['local_copy_dir']
 635     else:
 636         local_copy_dir = None
 637     if local_copy_dir is not None:
 638         fdroiddir = local_copy_dir.rstrip('/')
 639         if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
 640             logging.error(_('local_copy_dir must be directory, not a file!'))
 641             sys.exit(1)
 642         if not os.path.exists(os.path.dirname(fdroiddir)):
 643             logging.error(_('The root dir for local_copy_dir "{path}" does not exist!')
 644                           .format(path=os.path.dirname(fdroiddir)))
 645             sys.exit(1)
 646         if not os.path.isabs(fdroiddir):
 647             logging.error(_('local_copy_dir must be an absolute path!'))
 648             sys.exit(1)
 649         repobase = os.path.basename(fdroiddir)
 650         if standardwebroot and repobase != 'fdroid':
 651             logging.error(_('local_copy_dir does not end with "fdroid", '
 652                             + 'perhaps you meant: "{path}"')
 653                           .format(path=fdroiddir + '/fdroid'))
 654             sys.exit(1)
 655         if local_copy_dir[-1] != '/':
 656             local_copy_dir += '/'
 657         local_copy_dir = local_copy_dir.replace('//', '/')
 658         if not os.path.exists(fdroiddir):
 659             os.mkdir(fdroiddir)
 660
 661     if not config.get('awsbucket') \
 662             and not config.get('serverwebroot') \
 663             and not config.get('servergitmirrors') \
 664             and not config.get('androidobservatory') \
 665             and not config.get('binary_transparency_remote') \
 666             and not config.get('virustotal_apikey') \
 667             and local_copy_dir is None:
 668         logging.warn(_('No option set! Edit your config.py to set at least one of these:')
 669                      + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey, androidobservatory, or binary_transparency_remote')
 670         sys.exit(1)
 671
 672     repo_sections = ['repo']
 673     if config['archive_older'] != 0:
 674         repo_sections.append('archive')
 675         if not os.path.exists('archive'):
 676             os.mkdir('archive')
 677     if config['per_app_repos']:
 678         repo_sections += common.get_per_app_repos()
 679
 680     if options.command == 'init':
 681         ssh = paramiko.SSHClient()
 682         ssh.load_system_host_keys()
 683         for serverwebroot in config.get('serverwebroot', []):
 684             sshstr, remotepath = serverwebroot.rstrip('/').split(':')
 685             if sshstr.find('@') >= 0:
 686                 username, hostname = sshstr.split('@')
 687             else:
 688                 username = pwd.getpwuid(os.getuid())[0]  # get effective uid
 689                 hostname = sshstr
 690             ssh.connect(hostname, username=username)
 691             sftp = ssh.open_sftp()
 692             if os.path.basename(remotepath) \
 693                     not in sftp.listdir(os.path.dirname(remotepath)):
 694                 sftp.mkdir(remotepath, mode=0o755)
 695             for repo_section in repo_sections:
 696                 repo_path = os.path.join(remotepath, repo_section)
 697                 if os.path.basename(repo_path) \
 698                         not in sftp.listdir(remotepath):
 699                     sftp.mkdir(repo_path, mode=0o755)
 700             sftp.close()
 701             ssh.close()
 702     elif options.command == 'update':
 703         for repo_section in repo_sections:
 704             if local_copy_dir is not None:
 705                 if config['sync_from_local_copy_dir']:
 706                     sync_from_localcopy(repo_section, local_copy_dir)
 707                 else:
 708                     update_localcopy(repo_section, local_copy_dir)
 709             for serverwebroot in config.get('serverwebroot', []):
 710                 update_serverwebroot(serverwebroot, repo_section)
 711             if config.get('servergitmirrors', []):
 712                 # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
 713                 servergitmirrors = config.get('servergitmirrors', [])
 714                 update_servergitmirrors(servergitmirrors, repo_section)
 715             if config.get('awsbucket'):
 716                 update_awsbucket(repo_section)
 717             if config.get('androidobservatory'):
 718                 upload_to_android_observatory(repo_section)
 719             if config.get('virustotal_apikey'):
 720                 upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
 721
 722             binary_transparency_remote = config.get('binary_transparency_remote')
 723             if binary_transparency_remote:
 724                 push_binary_transparency(BINARY_TRANSPARENCY_DIR,
 725                                          binary_transparency_remote)
 726
 727     sys.exit(0)
 728
 729
 730 if __name__ == "__main__":
 731     main()