import threading
import magic
import logging
+import hashlib
+import socket
+
from distutils.version import LooseVersion
from zipfile import ZipFile
'stats_to_carbon': False,
'repo_maxage': 0,
'build_server_always': False,
- 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
+ 'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
'Summary': 50,
return exe_file
return None
+
+
+def genpassword():
+ '''generate a random password for when generating keys'''
+ h = hashlib.sha256()
+ h.update(os.urandom(16)) # salt
+ h.update(bytes(socket.getfqdn()))
+ return h.digest().encode('base64').strip()
+
+
+def genkeystore(localconfig):
+ '''Generate a new key with random passwords and add it to new keystore'''
+ logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
+ keystoredir = os.path.dirname(localconfig['keystore'])
+ if keystoredir is None or keystoredir == '':
+ keystoredir = os.path.join(os.getcwd(), keystoredir)
+ if not os.path.exists(keystoredir):
+ os.makedirs(keystoredir, mode=0o700)
+
+ write_password_file("keystorepass", localconfig['keystorepass'])
+ write_password_file("keypass", localconfig['keypass'])
+ p = FDroidPopen(['keytool', '-genkey',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-keyalg', 'RSA', '-keysize', '4096',
+ '-sigalg', 'SHA256withRSA',
+ '-validity', '10000',
+ '-storepass:file', config['keystorepassfile'],
+ '-keypass:file', config['keypassfile'],
+ '-dname', localconfig['keydname']])
+ # TODO keypass should be sent via stdin
+ os.chmod(localconfig['keystore'], 0o0600)
+ if p.returncode != 0:
+ raise BuildException("Failed to generate key", p.output)
+ # now show the lovely key that was just generated
+ p = FDroidPopen(['keytool', '-list', '-v',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:file', config['keystorepassfile']])
+ logging.info(p.output.strip() + '\n\n')
+
+
+def write_to_config(thisconfig, key, value=None):
+ '''write a key/value to the local config.py'''
+ if value is None:
+ origkey = key + '_orig'
+ value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
+ with open('config.py', 'r') as f:
+ data = f.read()
+ pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
+ repl = '\n' + key + ' = "' + value + '"'
+ data = re.sub(pattern, repl, data)
+ # if this key is not in the file, append it
+ if not re.match('\s*' + key + '\s*=\s*"', data):
+ data += repl
+ # make sure the file ends with a carraige return
+ if not re.match('\n$', data):
+ data += '\n'
+ with open('config.py', 'w') as f:
+ f.writelines(data)
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import glob
-import hashlib
import os
import re
import shutil
import logging
import common
-from common import FDroidPopen, BuildException
config = {}
options = None
-def write_to_config(thisconfig, key, value=None):
- '''write a key/value to the local config.py'''
- if value is None:
- origkey = key + '_orig'
- value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
- with open('config.py', 'r') as f:
- data = f.read()
- pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
- repl = '\n' + key + ' = "' + value + '"'
- data = re.sub(pattern, repl, data)
- with open('config.py', 'w') as f:
- f.writelines(data)
-
-
def disable_in_config(key, value):
'''write a key/value to the local config.py, then comment it out'''
with open('config.py', 'r') as f:
f.writelines(data)
-def genpassword():
- '''generate a random password for when generating keys'''
- h = hashlib.sha256()
- h.update(os.urandom(16)) # salt
- h.update(bytes(socket.getfqdn()))
- return h.digest().encode('base64').strip()
-
-
-def genkey(keystore, repo_keyalias, password, keydname):
- '''generate a new keystore with a new key in it for signing repos'''
- logging.info('Generating a new key in "' + keystore + '"...')
- common.write_password_file("keystorepass", password)
- common.write_password_file("keypass", password)
- p = FDroidPopen(['keytool', '-genkey',
- '-keystore', keystore, '-alias', repo_keyalias,
- '-keyalg', 'RSA', '-keysize', '4096',
- '-sigalg', 'SHA256withRSA',
- '-validity', '10000',
- '-storepass:file', config['keystorepassfile'],
- '-keypass:file', config['keypassfile'],
- '-dname', keydname])
- # TODO keypass should be sent via stdin
- if p.returncode != 0:
- raise BuildException("Failed to generate key", p.output)
- # now show the lovely key that was just generated
- p = FDroidPopen(['keytool', '-list', '-v',
- '-keystore', keystore, '-alias', repo_keyalias,
- '-storepass:file', config['keystorepassfile']])
- logging.info(p.output.strip() + '\n\n')
-
-
def main():
global options, config
# If android_home is not None, the path given from the command line
# will be directly written in the config.
if 'sdk_path' in test_config:
- write_to_config(test_config, 'sdk_path', options.android_home)
+ common.write_to_config(test_config, 'sdk_path', options.android_home)
else:
logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
logging.info('Try running `fdroid init` in an empty directory.')
test_config['build_tools'] = ''
else:
test_config['build_tools'] = dirname
- write_to_config(test_config, 'build_tools')
+ common.write_to_config(test_config, 'build_tools')
common.ensure_build_tools_exists(test_config)
# now that we have a local config.py, read configuration...
if not os.path.exists(keystore):
logging.info('"' + keystore
+ '" does not exist, creating a new keystore there.')
- write_to_config(test_config, 'keystore', keystore)
+ common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
- write_to_config(test_config, 'repo_keyalias', repo_keyalias)
+ common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
if options.distinguished_name:
keydname = options.distinguished_name
- write_to_config(test_config, 'keydname', keydname)
+ common.write_to_config(test_config, 'keydname', keydname)
if keystore == 'NONE': # we're using a smartcard
- write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
+ common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
disable_in_config('keypass', 'never used with smartcard')
- write_to_config(test_config, 'smartcardoptions',
- ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
- + '-providerClass sun.security.pkcs11.SunPKCS11 '
- + '-providerArg opensc-fdroid.cfg'))
+ common.write_to_config(test_config, 'smartcardoptions',
+ ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+ + '-providerClass sun.security.pkcs11.SunPKCS11 '
+ + '-providerArg opensc-fdroid.cfg'))
# find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'):
with open('opensc-fdroid.cfg', 'w') as f:
f.write(opensc_fdroid)
elif not os.path.exists(keystore):
- # no existing or specified keystore, generate the whole thing
- keystoredir = os.path.dirname(keystore)
- if not os.path.exists(keystoredir):
- os.makedirs(keystoredir, mode=0o700)
- password = genpassword()
- write_to_config(test_config, 'keystorepass', password)
- write_to_config(test_config, 'keypass', password)
- if options.repo_keyalias is None:
- repo_keyalias = socket.getfqdn()
- write_to_config(test_config, 'repo_keyalias', repo_keyalias)
- if not options.distinguished_name:
- keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
- write_to_config(test_config, 'keydname', keydname)
- genkey(keystore, repo_keyalias, password, keydname)
+ password = common.genpassword()
+ c = dict(test_config)
+ c['keystorepass'] = password
+ c['keypass'] = password
+ c['repo_keyalias'] = socket.getfqdn()
+ c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid'
+ common.write_to_config(test_config, 'keystorepass', password)
+ common.write_to_config(test_config, 'keypass', password)
+ common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias'])
+ common.write_to_config(test_config, 'keydname', c['keydname'])
+ common.genkeystore(c)
logging.info('Built repo based in "' + fdroiddir + '"')
logging.info('with this config:')
def update_serverwebroot(serverwebroot, repo_section):
# use a checksum comparison for accurate comparisons on different
# filesystems, for example, FAT has a low resolution timestamp
- rsyncargs = ['rsync', '--archive', '--delete', '--safe-links']
+ rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
if not options.no_checksum:
rsyncargs.append('--checksum')
if options.verbose:
rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
indexxml = os.path.join(repo_section, 'index.xml')
indexjar = os.path.join(repo_section, 'index.jar')
- # upload the first time without the index so that the repo stays working
- # while this update is running. Then once it is complete, rerun the
- # command again to upload the index. Always using the same target with
- # rsync allows for very strict settings on the receiving server, you can
- # literally specify the one rsync command that is allowed to run in
- # ~/.ssh/authorized_keys. (serverwebroot is guaranteed to have a trailing
- # slash in common.py)
+ # Upload the first time without the index files and delay the deletion as
+ # much as possible, that keeps the repo functional while this update is
+ # running. Then once it is complete, rerun the command again to upload
+ # the index files. Always using the same target with rsync allows for
+ # very strict settings on the receiving server, you can literally specify
+ # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
+ # (serverwebroot is guaranteed to have a trailing slash in common.py)
+ logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
if subprocess.call(rsyncargs +
['--exclude', indexxml, '--exclude', indexjar,
repo_section, serverwebroot]) != 0:
standardwebroot = True
for serverwebroot in config.get('serverwebroot', []):
- host, fdroiddir = serverwebroot.rstrip('/').split(':')
+ # this supports both an ssh host:path and just a path
+ s = serverwebroot.rstrip('/').split(':')
+ if len(s) == 1:
+ fdroiddir = s[0]
+ elif len(s) == 2:
+ host, fdroiddir = s
+ else:
+ logging.error('Malformed serverwebroot line: ' + serverwebroot)
+ sys.exit(1)
repobase = os.path.basename(fdroiddir)
if standardwebroot and repobase != 'fdroid':
logging.error('serverwebroot path does not end with "fdroid", '
import shutil
import glob
import re
+import socket
import zipfile
import hashlib
import pickle
repo_pubkey_fingerprint = None
+# Generate a certificate fingerprint the same way keytool does it
+# (but with slightly different formatting)
+def cert_fingerprint(data):
+ digest = hashlib.sha256(data).digest()
+ ret = []
+ ret.append(' '.join("%02X" % ord(b) for b in digest))
+ return " ".join(ret)
+
+
+def extract_pubkey():
+ global repo_pubkey_fingerprint
+ if 'repo_pubkey' in config:
+ pubkey = unhexlify(config['repo_pubkey'])
+ else:
+ p = FDroidPopen(['keytool', '-exportcert',
+ '-alias', config['repo_keyalias'],
+ '-keystore', config['keystore'],
+ '-storepass:file', config['keystorepassfile']]
+ + config['smartcardoptions'], output=False)
+ if p.returncode != 0 or len(p.output) < 20:
+ msg = "Failed to get repo pubkey!"
+ if config['keystore'] == 'NONE':
+ msg += ' Is your crypto smartcard plugged in?'
+ logging.critical(msg)
+ sys.exit(1)
+ pubkey = p.output
+ repo_pubkey_fingerprint = cert_fingerprint(pubkey)
+ return hexlify(pubkey)
+
+
def make_index(apps, sortedids, apks, repodir, archive, categories):
"""Make a repo index.
repoel.setAttribute("version", "12")
repoel.setAttribute("timestamp", str(int(time.time())))
- if 'repo_keyalias' in config:
-
- # Generate a certificate fingerprint the same way keytool does it
- # (but with slightly different formatting)
- def cert_fingerprint(data):
- digest = hashlib.sha256(data).digest()
- ret = []
- ret.append(' '.join("%02X" % ord(b) for b in digest))
- return " ".join(ret)
-
- def extract_pubkey():
- global repo_pubkey_fingerprint
- if 'repo_pubkey' in config:
- pubkey = unhexlify(config['repo_pubkey'])
- else:
- p = FDroidPopen(['keytool', '-exportcert',
- '-alias', config['repo_keyalias'],
- '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile']]
- + config['smartcardoptions'], output=False)
- if p.returncode != 0:
- msg = "Failed to get repo pubkey!"
- if config['keystore'] == 'NONE':
- msg += ' Is your crypto smartcard plugged in?'
- logging.critical(msg)
- sys.exit(1)
- pubkey = p.output
- repo_pubkey_fingerprint = cert_fingerprint(pubkey)
- return hexlify(pubkey)
-
- repoel.setAttribute("pubkey", extract_pubkey())
-
+ nosigningkey = False
+ if not 'repo_keyalias' in config:
+ nosigningkey = True
+ logging.critical("'repo_keyalias' not found in config.py!")
+ if not 'keystore' in config:
+ nosigningkey = True
+ logging.critical("'keystore' not found in config.py!")
+ if not 'keystorepass' in config:
+ nosigningkey = True
+ logging.critical("'keystorepass' not found in config.py!")
+ if not 'keypass' in config:
+ nosigningkey = True
+ logging.critical("'keypass' not found in config.py!")
+ if not os.path.exists(config['keystore']):
+ nosigningkey = True
+ logging.critical("'" + config['keystore'] + "' does not exist!")
+ if nosigningkey:
+ logging.warning("`fdroid update` requires a signing key, you can create one using:")
+ logging.warning("\tfdroid update --create-key")
+ sys.exit(1)
+
+ repoel.setAttribute("pubkey", extract_pubkey())
root.appendChild(repoel)
for appid in sortedids:
# Parse command line...
parser = OptionParser()
+ parser.add_option("--create-key", action="store_true", default=False,
+ help="Create a repo signing key in a keystore")
parser.add_option("-c", "--create-metadata", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("--delete-unknown", action="store_true", default=False,
logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
sys.exit(1)
+ # if the user asks to create a keystore, do it now, reusing whatever it can
+ if options.create_key:
+ if os.path.exists(config['keystore']):
+ logging.critical("Cowardily refusing to overwrite existing signing key setup!")
+ logging.critical("\t'" + config['keystore'] + "'")
+ sys.exit(1)
+
+ if not 'repo_keyalias' in config:
+ config['repo_keyalias'] = socket.getfqdn()
+ common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
+ if not 'keydname' in config:
+ config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
+ common.write_to_config(config, 'keydname', config['keydname'])
+ if not 'keystore' in config:
+ config['keystore'] = common.default_config.keystore
+ common.write_to_config(config, 'keystore', config['keystore'])
+
+ password = common.genpassword()
+ if not 'keystorepass' in config:
+ config['keystorepass'] = password
+ common.write_to_config(config, 'keystorepass', config['keystorepass'])
+ if not 'keypass' in config:
+ config['keypass'] = password
+ common.write_to_config(config, 'keypass', config['keypass'])
+ common.genkeystore(config)
+
# Get all apps...
apps = metadata.read_metadata()
#------------------------------------------------------------------------------#
# run local tests, don't scan fdroidserver/ project for APKs
+
+# this is a local repo on the Guardian Project Jenkins server
+apksource=/var/www/fdroid
+
cd $WORKSPACE/tests
-./run-tests ~jenkins/workspace/[[:upper:]a-eg-z]\*
+./run-tests $apksource
#------------------------------------------------------------------------------#
# run tests in new pip+virtualenv install
. $WORKSPACE/env/bin/activate
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
#------------------------------------------------------------------------------#
$fdroid init
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
LOCALCOPYDIR=`create_test_dir`/fdroid
$fdroid server update --local-copy-dir=$LOCALCOPYDIR
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
test -e repo/index.xml
test -e repo/index.jar
export ANDROID_HOME=$STORED_ANDROID_HOME
copy_apks_into_repo $REPOROOT
$fdroid init
$fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo manually and generate a keystore"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+touch config.py
+cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/
+! test -e $KEYSTORE
+set +e
+$fdroid update
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has no keystore!"
+ exit 1
+else
+ echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+$fdroid update --create-key
+test -e $KEYSTORE
+copy_apks_into_repo $REPOROOT
+$fdroid update --create-metadata
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
cp $WORKSPACE/tests/urzip.apk $REPOROOT/
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
test -e opensc-fdroid.cfg
test ! -e NONE
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo with no keystore, add APK, and update"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+touch config.py
+touch fdroid-icon.png
+mkdir repo/
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/
+set +e
+$fdroid update --create-metadata
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has no keystore!"
+ exit 1
+else
+ echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+
+# now set up fake, non-working keystore setup
+touch $KEYSTORE
+echo "keystore = \"$KEYSTORE\"" >> config.py
+echo 'repo_keyalias = "foo"' >> config.py
+echo 'keystorepass = "foo"' >> config.py
+echo 'keypass = "foo"' >> config.py
+set +e
+$fdroid update --create-metadata
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has a bad/fake keystore!"
+ exit 1
+else
+ echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo with keystore with APK, update, then without key"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE
+test -e $KEYSTORE
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
+$fdroid update --create-metadata
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+
+# now set fake repo_keyalias
+sed -i 's,^ *repo_keyalias.*,repo_keyalias = "fake",' $REPOROOT/config.py
+set +e
+$fdroid update
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has a bad repo_keyalias!"
+ exit 1
+else
+ echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+
+# try creating a new keystore, but fail because the old one is there
+test -e $KEYSTORE
+set +e
+$fdroid update --create-key
+if [ $? -eq 0 ]; then
+ echo "This should have failed because a keystore is already there!"
+ exit 1
+else
+ echo "`fdroid update` complained about existing keystore"
+fi
+set -e
+
+# now actually create the key with the existing settings
+rm -f $KEYSTORE
+! test -e $KEYSTORE
+$fdroid update --create-key
+test -e $KEYSTORE
+
+
+#------------------------------------------------------------------------------#
+
+# remove this to prevent git conflicts and complaining
rm -rf $WORKSPACE/fdroidserver.egg-info/
echo SUCCESS
~ianmdlvl / fdroidserver.git / commitdiff