3 # nightly.py - part of the FDroid server tools
4 # Copyright (C) 2017 Hans-Christoph Steiner <hans@eds.org>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from urllib.parse import urlparse
33 from argparse import ArgumentParser
39 # hard coded defaults for Android ~/.android/debug.keystore files
40 # https://developers.google.com/android/guides/client-auth
41 KEYSTORE_FILE = os.path.join(os.getenv('HOME'), '.android', 'debug.keystore')
43 KEY_ALIAS = 'androiddebugkey'
44 DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US'
46 # standard suffix for naming fdroid git repos
50 def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE):
51 tmp_dir = tempfile.mkdtemp(prefix='.')
52 privkey = os.path.join(tmp_dir, '.privkey')
53 key_pem = os.path.join(tmp_dir, '.key.pem')
54 p12 = os.path.join(tmp_dir, '.keystore.p12')
56 common.fill_config_defaults(_config)
57 subprocess.check_call([_config['keytool'], '-importkeystore',
58 '-srckeystore', keystore, '-srcalias', KEY_ALIAS,
59 '-srcstorepass', PASSWORD, '-srckeypass', PASSWORD,
60 '-destkeystore', p12, '-destalias', KEY_ALIAS,
61 '-deststorepass', PASSWORD, '-destkeypass', PASSWORD,
62 '-deststoretype', 'PKCS12'])
63 subprocess.check_call(['openssl', 'pkcs12', '-in', p12, '-out', key_pem,
64 '-passin', 'pass:' + PASSWORD, '-passout', 'pass:' + PASSWORD])
65 subprocess.check_call(['openssl', 'rsa', '-in', key_pem, '-out', privkey,
66 '-passin', 'pass:' + PASSWORD])
69 os.chmod(privkey, 0o600) # os.umask() should cover this, but just in case
71 rsakey = paramiko.RSAKey.from_private_key_file(privkey)
72 fingerprint = base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()).decode('ascii').rstrip('=')
73 ssh_private_key_file = os.path.join(tmp_dir, 'debug_keystore_'
74 + fingerprint.replace('/', '_') + '_id_rsa')
75 shutil.move(privkey, ssh_private_key_file)
77 pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file
78 with open(ssh_private_key_file + '.pub', 'w') as fp:
81 logging.info(_('\nSSH Public Key to be used as Deploy Key:') + '\n' + pub)
83 return ssh_private_key_file
88 parser = ArgumentParser(usage="%(prog)s")
89 common.setup_global_opts(parser)
90 parser.add_argument("--keystore", default=KEYSTORE_FILE,
91 help=_("Specify which debug keystore file to use."))
92 parser.add_argument("--show-secret-var", action="store_true", default=False,
93 help=_("Print the secret variable to the terminal for easy copy/paste"))
94 parser.add_argument("--file", default='app/build/outputs/apk/*.apk',
95 help=_('The the file to be included in the repo (path or glob)'))
96 parser.add_argument("--no-checksum", action="store_true", default=False,
97 help=_("Don't use rsync checksums"))
98 # TODO add --with-btlog
99 options = parser.parse_args()
101 # force a tighter umask since this writes private key material
102 umask = os.umask(0o077)
104 if 'CI' in os.environ:
105 v = os.getenv('DEBUG_KEYSTORE')
106 debug_keystore = None
108 debug_keystore = base64.b64decode(v)
109 if not debug_keystore:
110 logging.error(_('DEBUG_KEYSTORE is not set or the value is incomplete'))
112 os.makedirs(os.path.dirname(KEYSTORE_FILE), exist_ok=True)
113 if os.path.exists(KEYSTORE_FILE):
114 logging.warning(_('overwriting existing {path}').format(path=KEYSTORE_FILE))
115 with open(KEYSTORE_FILE, 'wb') as fp:
116 fp.write(debug_keystore)
118 repo_basedir = os.path.join(os.getcwd(), 'fdroid')
119 repodir = os.path.join(repo_basedir, 'repo')
121 os.makedirs(repodir, exist_ok=True)
123 if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ:
124 # we are in GitLab CI
125 repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY
126 clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY
127 repo_base = clone_url + '/raw/master/fdroid'
128 servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
129 deploy_key_url = clone_url + '/settings/repository'
130 git_user_name = os.getenv('GITLAB_USER_NAME')
131 git_user_email = os.getenv('GITLAB_USER_EMAIL')
132 elif 'TRAVIS_REPO_SLUG' in os.environ:
133 # we are in Travis CI
134 repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY
135 clone_url = 'https://github.com/' + repo_git_base
136 _branch = os.getenv('TRAVIS_BRANCH')
137 repo_base = 'https://raw.githubusercontent.com/' + repo_git_base + '/' + _branch + '/fdroid'
138 servergitmirror = 'git@github.com:' + repo_git_base
139 deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys'
140 + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
141 git_user_name = repo_git_base
142 git_user_email = os.getenv('USER') + '@' + platform.node()
143 elif 'CIRCLE_REPOSITORY_URL' in os.environ \
144 and 'CIRCLE_PROJECT_USERNAME' in os.environ \
145 and 'CIRCLE_PROJECT_REPONAME' in os.environ:
146 # we are in Circle CI
147 repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME')
148 + '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY)
149 clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
150 repo_base = clone_url + '/raw/master/fdroid'
151 servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
152 deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys'
153 + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
154 git_user_name = os.getenv('CIRCLE_USERNAME')
155 git_user_email = git_user_name + '@' + platform.node()
157 print(_('ERROR: unsupported CI type, patches welcome!'))
160 repo_url = repo_base + '/repo'
161 git_mirror_path = os.path.join(repo_basedir, 'git-mirror')
162 git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo')
163 git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata')
164 if not os.path.isdir(git_mirror_repodir):
165 logging.debug(_('cloning {url}').format(url=clone_url))
167 git.Repo.clone_from(clone_url, git_mirror_path)
170 if not os.path.isdir(git_mirror_repodir):
171 os.makedirs(git_mirror_repodir, mode=0o755)
173 mirror_git_repo = git.Repo.init(git_mirror_path)
174 writer = mirror_git_repo.config_writer()
175 writer.set_value('user', 'name', git_user_name)
176 writer.set_value('user', 'email', git_user_email)
178 for remote in mirror_git_repo.remotes:
179 mirror_git_repo.delete_remote(remote)
181 readme_path = os.path.join(git_mirror_path, 'README.md')
185 [![{repo_url}](icon.png)]({repo_url})
187 Last updated: {date}'''.format(repo_git_base=repo_git_base,
189 date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'))
190 with open(readme_path, 'w') as fp:
192 mirror_git_repo.git.add(all=True)
193 mirror_git_repo.index.commit("update README")
195 icon_path = os.path.join(git_mirror_path, 'icon.png')
198 qrcode.make(repo_url).save(icon_path)
200 exampleicon = os.path.join(common.get_examples_dir(), 'fdroid-icon.png')
201 shutil.copy(exampleicon, icon_path)
202 mirror_git_repo.git.add(all=True)
203 mirror_git_repo.index.commit("update repo/website icon")
204 shutil.copy(icon_path, repo_basedir)
206 os.chdir(repo_basedir)
207 if os.path.isdir(git_mirror_repodir):
208 common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
209 if os.path.isdir(git_mirror_metadatadir):
210 common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
212 ssh_private_key_file = _ssh_key_from_debug_keystore()
213 # this is needed for GitPython to find the SSH key
214 ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
215 os.makedirs(ssh_dir, exist_ok=True)
216 ssh_config = os.path.join(ssh_dir, 'config')
217 logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config))
218 with open(ssh_config, 'a') as fp:
219 fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file)
222 config += "identity_file = '%s'\n" % ssh_private_key_file
223 config += "repo_name = '%s'\n" % repo_git_base
224 config += "repo_url = '%s'\n" % repo_url
225 config += "repo_icon = 'icon.png'\n"
226 config += "archive_name = '%s'\n" % (repo_git_base + ' archive')
227 config += "archive_url = '%s'\n" % (repo_base + '/archive')
228 config += "archive_icon = 'icon.png'\n"
229 config += "servergitmirrors = '%s'\n" % servergitmirror
230 config += "keystore = '%s'\n" % KEYSTORE_FILE
231 config += "repo_keyalias = '%s'\n" % KEY_ALIAS
232 config += "keystorepass = '%s'\n" % PASSWORD
233 config += "keypass = '%s'\n" % PASSWORD
234 config += "keydname = '%s'\n" % DISTINGUISHED_NAME
235 config += "make_current_version_link = False\n"
236 config += "accepted_formats = ('txt', 'yml')\n"
237 # TODO add update_stats = True
238 with open('config.py', 'w') as fp:
240 os.chmod('config.py', 0o600)
241 config = common.read_config(options)
242 common.assert_config_keystore(config)
244 for root, dirs, files in os.walk(cibase):
245 for d in ('fdroid', '.git', '.gradle'):
249 if f.endswith('-debug.apk'):
250 apkfilename = os.path.join(root, f)
251 logging.debug(_('Striping mystery signature from {apkfilename}')
252 .format(apkfilename=apkfilename))
253 destapk = os.path.join(repodir, os.path.basename(f))
254 os.chmod(apkfilename, 0o644)
255 logging.debug(_('Resigning {apkfilename} with provided debug.keystore')
256 .format(apkfilename=os.path.basename(apkfilename)))
257 common.apk_strip_signatures(apkfilename, strip_manifest=True)
258 common.sign_apk(apkfilename, destapk, KEY_ALIAS)
261 logging.debug(_('attempting bare ssh connection to test deploy key:'))
263 subprocess.check_call(['ssh', '-Tvi', ssh_private_key_file,
264 '-oIdentitiesOnly=yes', '-oStrictHostKeyChecking=no',
265 servergitmirror.split(':')[0]])
266 except subprocess.CalledProcessError:
269 app_url = clone_url[:-len(NIGHTLY)]
271 template['AuthorName'] = clone_url.split('/')[4]
272 template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4])
273 template['Categories'] = ['nightly']
274 template['SourceCode'] = app_url
275 template['IssueTracker'] = app_url + '/issues'
276 template['Summary'] = 'Nightly build of ' + urlparse(app_url).path[1:]
277 template['Description'] = template['Summary']
278 with open('template.yml', 'w') as fp:
279 yaml.dump(template, fp)
281 subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
283 common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/')
284 mirror_git_repo.git.add(all=True)
285 mirror_git_repo.index.commit("update app metadata")
287 subprocess.check_call(['fdroid', 'server', 'update', '--verbose'], cwd=repo_basedir)
288 except subprocess.CalledProcessError:
289 logging.error(_('cannot publish update, did you set the deploy key?')
290 + '\n' + deploy_key_url)
292 if shutil.rmtree.avoids_symlink_attacks:
293 shutil.rmtree(os.path.dirname(ssh_private_key_file))
296 if not os.path.isfile(options.keystore):
297 androiddir = os.path.dirname(options.keystore)
298 if not os.path.exists(androiddir):
300 logging.info(_('created {path}').format(path=androiddir))
301 logging.error(_('{path} does not exist! Create it by running:').format(path=options.keystore)
302 + '\n keytool -genkey -v -keystore ' + options.keystore + ' -storepass android \\'
303 + '\n -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\'
304 + '\n -dname "CN=Android Debug,O=Android,C=US"')
306 ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
307 os.makedirs(os.path.dirname(ssh_dir), exist_ok=True)
308 privkey = _ssh_key_from_debug_keystore(options.keystore)
309 ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))
310 shutil.move(privkey, ssh_private_key_file)
311 shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')
312 if shutil.rmtree.avoids_symlink_attacks:
313 shutil.rmtree(os.path.dirname(privkey))
315 if options.show_secret_var:
316 with open(options.keystore, 'rb') as fp:
317 debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii')
318 print(_('\n{path} encoded for the DEBUG_KEYSTORE secret variable:')
319 .format(path=options.keystore))
320 print(debug_keystore)
325 if __name__ == "__main__":