3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from binascii import hexlify
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 from pyasn1.codec.der import decoder, encoder
46 from pyasn1_modules import rfc2315
47 from pyasn1.error import PyAsn1Error
49 from distutils.util import strtobool
51 import fdroidserver.metadata
52 from fdroidserver.exception import FDroidException, VCSException, BuildException
53 from .asynchronousfilereader import AsynchronousFileReader
56 # A signature block file with a .DSA, .RSA, or .EC extension
57 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
58 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
59 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
61 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
70 'sdk_path': "$ANDROID_HOME",
75 'r12b': "$ANDROID_NDK",
81 'build_tools': "25.0.2",
82 'force_build_tools': False,
87 'accepted_formats': ['txt', 'yml'],
88 'sync_from_local_copy_dir': False,
89 'allow_disabled_algorithms': False,
90 'per_app_repos': False,
91 'make_current_version_link': True,
92 'current_version_name_source': 'Name',
93 'update_stats': False,
97 'stats_to_carbon': False,
99 'build_server_always': False,
100 'keystore': 'keystore.jks',
101 'smartcardoptions': [],
111 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
112 'repo_name': "My First FDroid Repo Demo",
113 'repo_icon': "fdroid-icon.png",
114 'repo_description': '''
115 This is a repository of apps to be used with FDroid. Applications in this
116 repository are either official binaries built by the original application
117 developers, or are binaries built from source by the admin of f-droid.org
118 using the tools on https://gitlab.com/u/fdroid.
124 def setup_global_opts(parser):
125 parser.add_argument("-v", "--verbose", action="store_true", default=False,
126 help="Spew out even more information than normal")
127 parser.add_argument("-q", "--quiet", action="store_true", default=False,
128 help="Restrict output to warnings and errors")
131 def fill_config_defaults(thisconfig):
132 for k, v in default_config.items():
133 if k not in thisconfig:
136 # Expand paths (~users and $vars)
137 def expand_path(path):
141 path = os.path.expanduser(path)
142 path = os.path.expandvars(path)
147 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
152 thisconfig[k + '_orig'] = v
154 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
155 if thisconfig['java_paths'] is None:
156 thisconfig['java_paths'] = dict()
158 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
159 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
160 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
161 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
162 if os.getenv('JAVA_HOME') is not None:
163 pathlist.append(os.getenv('JAVA_HOME'))
164 if os.getenv('PROGRAMFILES') is not None:
165 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
166 for d in sorted(pathlist):
167 if os.path.islink(d):
169 j = os.path.basename(d)
170 # the last one found will be the canonical one, so order appropriately
172 r'^1\.([6-9])\.0\.jdk$', # OSX
173 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
174 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
175 r'^jdk([6-9])-openjdk$', # Arch
176 r'^java-([6-9])-openjdk$', # Arch
177 r'^java-([6-9])-jdk$', # Arch (oracle)
178 r'^java-1\.([6-9])\.0-.*$', # RedHat
179 r'^java-([6-9])-oracle$', # Debian WebUpd8
180 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
181 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
183 m = re.match(regex, j)
186 for p in [d, os.path.join(d, 'Contents', 'Home')]:
187 if os.path.exists(os.path.join(p, 'bin', 'javac')):
188 thisconfig['java_paths'][m.group(1)] = p
190 for java_version in ('7', '8', '9'):
191 if java_version not in thisconfig['java_paths']:
193 java_home = thisconfig['java_paths'][java_version]
194 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
195 if os.path.exists(jarsigner):
196 thisconfig['jarsigner'] = jarsigner
197 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
198 break # Java7 is preferred, so quit if found
200 for k in ['ndk_paths', 'java_paths']:
206 thisconfig[k][k2] = exp
207 thisconfig[k][k2 + '_orig'] = v
210 def regsub_file(pattern, repl, path):
211 with open(path, 'rb') as f:
213 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
214 with open(path, 'wb') as f:
218 def read_config(opts, config_file='config.py'):
219 """Read the repository config
221 The config is read from config_file, which is in the current
222 directory when any of the repo management commands are used. If
223 there is a local metadata file in the git repo, then config.py is
224 not required, just use defaults.
227 global config, options
229 if config is not None:
236 if os.path.isfile(config_file):
237 logging.debug("Reading %s" % config_file)
238 with io.open(config_file, "rb") as f:
239 code = compile(f.read(), config_file, 'exec')
240 exec(code, None, config)
241 elif len(get_local_metadata_files()) == 0:
242 raise FDroidException("Missing config file - is this a repo directory?")
244 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
246 if not type(config[k]) in (str, list, tuple):
247 logging.warn('"' + k + '" will be in random order!'
248 + ' Use () or [] brackets if order is important!')
250 # smartcardoptions must be a list since its command line args for Popen
251 if 'smartcardoptions' in config:
252 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
253 elif 'keystore' in config and config['keystore'] == 'NONE':
254 # keystore='NONE' means use smartcard, these are required defaults
255 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
256 'SunPKCS11-OpenSC', '-providerClass',
257 'sun.security.pkcs11.SunPKCS11',
258 '-providerArg', 'opensc-fdroid.cfg']
260 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
261 st = os.stat(config_file)
262 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
263 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
265 fill_config_defaults(config)
267 for k in ["repo_description", "archive_description"]:
269 config[k] = clean_description(config[k])
271 if 'serverwebroot' in config:
272 if isinstance(config['serverwebroot'], str):
273 roots = [config['serverwebroot']]
274 elif all(isinstance(item, str) for item in config['serverwebroot']):
275 roots = config['serverwebroot']
277 raise TypeError('only accepts strings, lists, and tuples')
279 for rootstr in roots:
280 # since this is used with rsync, where trailing slashes have
281 # meaning, ensure there is always a trailing slash
282 if rootstr[-1] != '/':
284 rootlist.append(rootstr.replace('//', '/'))
285 config['serverwebroot'] = rootlist
287 if 'servergitmirrors' in config:
288 if isinstance(config['servergitmirrors'], str):
289 roots = [config['servergitmirrors']]
290 elif all(isinstance(item, str) for item in config['servergitmirrors']):
291 roots = config['servergitmirrors']
293 raise TypeError('only accepts strings, lists, and tuples')
294 config['servergitmirrors'] = roots
299 def find_sdk_tools_cmd(cmd):
300 '''find a working path to a tool from the Android SDK'''
303 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
304 # try to find a working path to this command, in all the recent possible paths
305 if 'build_tools' in config:
306 build_tools = os.path.join(config['sdk_path'], 'build-tools')
307 # if 'build_tools' was manually set and exists, check only that one
308 configed_build_tools = os.path.join(build_tools, config['build_tools'])
309 if os.path.exists(configed_build_tools):
310 tooldirs.append(configed_build_tools)
312 # no configed version, so hunt known paths for it
313 for f in sorted(os.listdir(build_tools), reverse=True):
314 if os.path.isdir(os.path.join(build_tools, f)):
315 tooldirs.append(os.path.join(build_tools, f))
316 tooldirs.append(build_tools)
317 sdk_tools = os.path.join(config['sdk_path'], 'tools')
318 if os.path.exists(sdk_tools):
319 tooldirs.append(sdk_tools)
320 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
321 if os.path.exists(sdk_platform_tools):
322 tooldirs.append(sdk_platform_tools)
323 tooldirs.append('/usr/bin')
325 path = os.path.join(d, cmd)
326 if os.path.isfile(path):
328 test_aapt_version(path)
330 # did not find the command, exit with error message
331 ensure_build_tools_exists(config)
334 def test_aapt_version(aapt):
335 '''Check whether the version of aapt is new enough'''
336 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
337 if output is None or output == '':
338 logging.error(aapt + ' failed to execute!')
340 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
345 # the Debian package has the version string like "v0.2-23.0.2"
346 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
347 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
349 logging.warning('Unknown version of aapt, might cause problems: ' + output)
352 def test_sdk_exists(thisconfig):
353 if 'sdk_path' not in thisconfig:
354 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
355 test_aapt_version(thisconfig['aapt'])
358 logging.error("'sdk_path' not set in config.py!")
360 if thisconfig['sdk_path'] == default_config['sdk_path']:
361 logging.error('No Android SDK found!')
362 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
363 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
365 if not os.path.exists(thisconfig['sdk_path']):
366 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
368 if not os.path.isdir(thisconfig['sdk_path']):
369 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
371 for d in ['build-tools', 'platform-tools', 'tools']:
372 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
373 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
374 thisconfig['sdk_path'], d))
379 def ensure_build_tools_exists(thisconfig):
380 if not test_sdk_exists(thisconfig):
381 raise FDroidException("Android SDK not found.")
382 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
383 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
384 if not os.path.isdir(versioned_build_tools):
385 raise FDroidException(
386 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
389 def get_local_metadata_files():
390 '''get any metadata files local to an app's source repo
392 This tries to ignore anything that does not count as app metdata,
393 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
396 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
399 def read_pkg_args(args, allow_vercodes=False):
401 :param args: arguments in the form of multiple appid:[vc] strings
402 :returns: a dictionary with the set of vercodes specified for each package
410 if allow_vercodes and ':' in p:
411 package, vercode = p.split(':')
413 package, vercode = p, None
414 if package not in vercodes:
415 vercodes[package] = [vercode] if vercode else []
417 elif vercode and vercode not in vercodes[package]:
418 vercodes[package] += [vercode] if vercode else []
423 def read_app_args(args, allapps, allow_vercodes=False):
425 On top of what read_pkg_args does, this returns the whole app metadata, but
426 limiting the builds list to the builds matching the vercodes specified.
429 vercodes = read_pkg_args(args, allow_vercodes)
435 for appid, app in allapps.items():
436 if appid in vercodes:
439 if len(apps) != len(vercodes):
442 logging.critical("No such package: %s" % p)
443 raise FDroidException("Found invalid app ids in arguments")
445 raise FDroidException("No packages specified")
448 for appid, app in apps.items():
452 app.builds = [b for b in app.builds if b.versionCode in vc]
453 if len(app.builds) != len(vercodes[appid]):
455 allvcs = [b.versionCode for b in app.builds]
456 for v in vercodes[appid]:
458 logging.critical("No such vercode %s for app %s" % (v, appid))
461 raise FDroidException("Found invalid vercodes for some apps")
466 def get_extension(filename):
467 base, ext = os.path.splitext(filename)
470 return base, ext.lower()[1:]
473 def has_extension(filename, ext):
474 _, f_ext = get_extension(filename)
478 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
481 def clean_description(description):
482 'Remove unneeded newlines and spaces from a block of description text'
484 # this is split up by paragraph to make removing the newlines easier
485 for paragraph in re.split(r'\n\n', description):
486 paragraph = re.sub('\r', '', paragraph)
487 paragraph = re.sub('\n', ' ', paragraph)
488 paragraph = re.sub(' {2,}', ' ', paragraph)
489 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
490 returnstring += paragraph + '\n\n'
491 return returnstring.rstrip('\n')
494 def publishednameinfo(filename):
495 filename = os.path.basename(filename)
496 m = publish_name_regex.match(filename)
498 result = (m.group(1), m.group(2))
499 except AttributeError:
500 raise FDroidException("Invalid name for published file: %s" % filename)
504 def get_release_filename(app, build):
506 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
508 return "%s_%s.apk" % (app.id, build.versionCode)
511 def get_toolsversion_logname(app, build):
512 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
515 def getsrcname(app, build):
516 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
528 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
531 def get_build_dir(app):
532 '''get the dir that this app will be built in'''
534 if app.RepoType == 'srclib':
535 return os.path.join('build', 'srclib', app.Repo)
537 return os.path.join('build', app.id)
541 '''checkout code from VCS and return instance of vcs and the build dir'''
542 build_dir = get_build_dir(app)
544 # Set up vcs interface and make sure we have the latest code...
545 logging.debug("Getting {0} vcs interface for {1}"
546 .format(app.RepoType, app.Repo))
547 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
551 vcs = getvcs(app.RepoType, remote, build_dir)
553 return vcs, build_dir
556 def getvcs(vcstype, remote, local):
558 return vcs_git(remote, local)
559 if vcstype == 'git-svn':
560 return vcs_gitsvn(remote, local)
562 return vcs_hg(remote, local)
564 return vcs_bzr(remote, local)
565 if vcstype == 'srclib':
566 if local != os.path.join('build', 'srclib', remote):
567 raise VCSException("Error: srclib paths are hard-coded!")
568 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
570 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
571 raise VCSException("Invalid vcs type " + vcstype)
574 def getsrclibvcs(name):
575 if name not in fdroidserver.metadata.srclibs:
576 raise VCSException("Missing srclib " + name)
577 return fdroidserver.metadata.srclibs[name]['Repo Type']
582 def __init__(self, remote, local):
584 # svn, git-svn and bzr may require auth
586 if self.repotype() in ('git-svn', 'bzr'):
588 if self.repotype == 'git-svn':
589 raise VCSException("Authentication is not supported for git-svn")
590 self.username, remote = remote.split('@')
591 if ':' not in self.username:
592 raise VCSException("Password required with username")
593 self.username, self.password = self.username.split(':')
597 self.clone_failed = False
598 self.refreshed = False
604 # Take the local repository to a clean version of the given revision, which
605 # is specificed in the VCS's native format. Beforehand, the repository can
606 # be dirty, or even non-existent. If the repository does already exist
607 # locally, it will be updated from the origin, but only once in the
608 # lifetime of the vcs object.
609 # None is acceptable for 'rev' if you know you are cloning a clean copy of
610 # the repo - otherwise it must specify a valid revision.
611 def gotorevision(self, rev, refresh=True):
613 if self.clone_failed:
614 raise VCSException("Downloading the repository already failed once, not trying again.")
616 # The .fdroidvcs-id file for a repo tells us what VCS type
617 # and remote that directory was created from, allowing us to drop it
618 # automatically if either of those things changes.
619 fdpath = os.path.join(self.local, '..',
620 '.fdroidvcs-' + os.path.basename(self.local))
621 fdpath = os.path.normpath(fdpath)
622 cdata = self.repotype() + ' ' + self.remote
625 if os.path.exists(self.local):
626 if os.path.exists(fdpath):
627 with open(fdpath, 'r') as f:
628 fsdata = f.read().strip()
633 logging.info("Repository details for %s changed - deleting" % (
637 logging.info("Repository details for %s missing - deleting" % (
640 shutil.rmtree(self.local)
644 self.refreshed = True
647 self.gotorevisionx(rev)
648 except FDroidException as e:
651 # If necessary, write the .fdroidvcs file.
652 if writeback and not self.clone_failed:
653 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
654 with open(fdpath, 'w+') as f:
660 # Derived classes need to implement this. It's called once basic checking
661 # has been performend.
662 def gotorevisionx(self, rev): # pylint: disable=unused-argument
663 raise VCSException("This VCS type doesn't define gotorevisionx")
665 # Initialise and update submodules
666 def initsubmodules(self):
667 raise VCSException('Submodules not supported for this vcs type')
669 # Get a list of all known tags
671 if not self._gettags:
672 raise VCSException('gettags not supported for this vcs type')
674 for tag in self._gettags():
675 if re.match('[-A-Za-z0-9_. /]+$', tag):
679 # Get a list of all the known tags, sorted from newest to oldest
680 def latesttags(self):
681 raise VCSException('latesttags not supported for this vcs type')
683 # Get current commit reference (hash, revision, etc)
685 raise VCSException('getref not supported for this vcs type')
687 # Returns the srclib (name, path) used in setting up the current
698 # If the local directory exists, but is somehow not a git repository, git
699 # will traverse up the directory tree until it finds one that is (i.e.
700 # fdroidserver) and then we'll proceed to destroy it! This is called as
703 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
704 result = p.output.rstrip()
705 if not result.endswith(self.local):
706 raise VCSException('Repository mismatch')
708 def gotorevisionx(self, rev):
709 if not os.path.exists(self.local):
711 p = FDroidPopen(['git', 'clone', self.remote, self.local])
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git clone failed", p.output)
718 # Discard any working tree changes
719 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
720 'git', 'reset', '--hard'], cwd=self.local, output=False)
721 if p.returncode != 0:
722 raise VCSException("Git reset failed", p.output)
723 # Remove untracked files now, in case they're tracked in the target
724 # revision (it happens!)
725 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
726 'git', 'clean', '-dffx'], cwd=self.local, output=False)
727 if p.returncode != 0:
728 raise VCSException("Git clean failed", p.output)
729 if not self.refreshed:
730 # Get latest commits and tags from remote
731 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
732 if p.returncode != 0:
733 raise VCSException("Git fetch failed", p.output)
734 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
735 if p.returncode != 0:
736 raise VCSException("Git fetch failed", p.output)
737 # Recreate origin/HEAD as git clone would do it, in case it disappeared
738 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 lines = p.output.splitlines()
741 if 'Multiple remote HEAD branches' not in lines[0]:
742 raise VCSException("Git remote set-head failed", p.output)
743 branch = lines[1].split(' ')[-1]
744 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
745 if p2.returncode != 0:
746 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
747 self.refreshed = True
748 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
749 # a github repo. Most of the time this is the same as origin/master.
750 rev = rev or 'origin/HEAD'
751 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
752 if p.returncode != 0:
753 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
754 # Get rid of any uncontrolled files left behind
755 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
756 if p.returncode != 0:
757 raise VCSException("Git clean failed", p.output)
759 def initsubmodules(self):
761 submfile = os.path.join(self.local, '.gitmodules')
762 if not os.path.isfile(submfile):
763 raise VCSException("No git submodules available")
765 # fix submodules not accessible without an account and public key auth
766 with open(submfile, 'r') as f:
767 lines = f.readlines()
768 with open(submfile, 'w') as f:
770 if 'git@github.com' in line:
771 line = line.replace('git@github.com:', 'https://github.com/')
772 if 'git@gitlab.com' in line:
773 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
776 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
777 if p.returncode != 0:
778 raise VCSException("Git submodule sync failed", p.output)
779 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
780 if p.returncode != 0:
781 raise VCSException("Git submodule update failed", p.output)
785 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
786 return p.output.splitlines()
788 tag_format = re.compile(r'tag: ([^),]*)')
790 def latesttags(self):
792 p = FDroidPopen(['git', 'log', '--tags',
793 '--simplify-by-decoration', '--pretty=format:%d'],
794 cwd=self.local, output=False)
796 for line in p.output.splitlines():
797 for tag in self.tag_format.findall(line):
802 class vcs_gitsvn(vcs):
807 # If the local directory exists, but is somehow not a git repository, git
808 # will traverse up the directory tree until it finds one that is (i.e.
809 # fdroidserver) and then we'll proceed to destory it! This is called as
812 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
813 result = p.output.rstrip()
814 if not result.endswith(self.local):
815 raise VCSException('Repository mismatch')
817 def gotorevisionx(self, rev):
818 if not os.path.exists(self.local):
820 gitsvn_args = ['git', 'svn', 'clone']
821 if ';' in self.remote:
822 remote_split = self.remote.split(';')
823 for i in remote_split[1:]:
824 if i.startswith('trunk='):
825 gitsvn_args.extend(['-T', i[6:]])
826 elif i.startswith('tags='):
827 gitsvn_args.extend(['-t', i[5:]])
828 elif i.startswith('branches='):
829 gitsvn_args.extend(['-b', i[9:]])
830 gitsvn_args.extend([remote_split[0], self.local])
831 p = FDroidPopen(gitsvn_args, output=False)
832 if p.returncode != 0:
833 self.clone_failed = True
834 raise VCSException("Git svn clone failed", p.output)
836 gitsvn_args.extend([self.remote, self.local])
837 p = FDroidPopen(gitsvn_args, output=False)
838 if p.returncode != 0:
839 self.clone_failed = True
840 raise VCSException("Git svn clone failed", p.output)
844 # Discard any working tree changes
845 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
846 if p.returncode != 0:
847 raise VCSException("Git reset failed", p.output)
848 # Remove untracked files now, in case they're tracked in the target
849 # revision (it happens!)
850 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
851 if p.returncode != 0:
852 raise VCSException("Git clean failed", p.output)
853 if not self.refreshed:
854 # Get new commits, branches and tags from repo
855 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
856 if p.returncode != 0:
857 raise VCSException("Git svn fetch failed")
858 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
859 if p.returncode != 0:
860 raise VCSException("Git svn rebase failed", p.output)
861 self.refreshed = True
863 rev = rev or 'master'
865 nospaces_rev = rev.replace(' ', '%20')
866 # Try finding a svn tag
867 for treeish in ['origin/', '']:
868 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
869 if p.returncode == 0:
871 if p.returncode != 0:
872 # No tag found, normal svn rev translation
873 # Translate svn rev into git format
874 rev_split = rev.split('/')
877 for treeish in ['origin/', '']:
878 if len(rev_split) > 1:
879 treeish += rev_split[0]
880 svn_rev = rev_split[1]
883 # if no branch is specified, then assume trunk (i.e. 'master' branch):
887 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
889 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
890 git_rev = p.output.rstrip()
892 if p.returncode == 0 and git_rev:
895 if p.returncode != 0 or not git_rev:
896 # Try a plain git checkout as a last resort
897 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
898 if p.returncode != 0:
899 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
901 # Check out the git rev equivalent to the svn rev
902 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
903 if p.returncode != 0:
904 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
906 # Get rid of any uncontrolled files left behind
907 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
908 if p.returncode != 0:
909 raise VCSException("Git clean failed", p.output)
913 for treeish in ['origin/', '']:
914 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
920 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
921 if p.returncode != 0:
923 return p.output.strip()
931 def gotorevisionx(self, rev):
932 if not os.path.exists(self.local):
933 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
934 if p.returncode != 0:
935 self.clone_failed = True
936 raise VCSException("Hg clone failed", p.output)
938 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
939 if p.returncode != 0:
940 raise VCSException("Hg status failed", p.output)
941 for line in p.output.splitlines():
942 if not line.startswith('? '):
943 raise VCSException("Unexpected output from hg status -uS: " + line)
944 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
945 if not self.refreshed:
946 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
947 if p.returncode != 0:
948 raise VCSException("Hg pull failed", p.output)
949 self.refreshed = True
951 rev = rev or 'default'
954 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
955 if p.returncode != 0:
956 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
957 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
958 # Also delete untracked files, we have to enable purge extension for that:
959 if "'purge' is provided by the following extension" in p.output:
960 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
961 myfile.write("\n[extensions]\nhgext.purge=\n")
962 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
963 if p.returncode != 0:
964 raise VCSException("HG purge failed", p.output)
965 elif p.returncode != 0:
966 raise VCSException("HG purge failed", p.output)
969 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
970 return p.output.splitlines()[1:]
978 def gotorevisionx(self, rev):
979 if not os.path.exists(self.local):
980 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
981 if p.returncode != 0:
982 self.clone_failed = True
983 raise VCSException("Bzr branch failed", p.output)
985 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
986 if p.returncode != 0:
987 raise VCSException("Bzr revert failed", p.output)
988 if not self.refreshed:
989 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
990 if p.returncode != 0:
991 raise VCSException("Bzr update failed", p.output)
992 self.refreshed = True
994 revargs = list(['-r', rev] if rev else [])
995 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
996 if p.returncode != 0:
997 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1000 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1001 return [tag.split(' ')[0].strip() for tag in
1002 p.output.splitlines()]
1005 def unescape_string(string):
1008 if string[0] == '"' and string[-1] == '"':
1011 return string.replace("\\'", "'")
1014 def retrieve_string(app_dir, string, xmlfiles=None):
1016 if not string.startswith('@string/'):
1017 return unescape_string(string)
1019 if xmlfiles is None:
1022 os.path.join(app_dir, 'res'),
1023 os.path.join(app_dir, 'src', 'main', 'res'),
1025 for r, d, f in os.walk(res_dir):
1026 if os.path.basename(r) == 'values':
1027 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1029 name = string[len('@string/'):]
1031 def element_content(element):
1032 if element.text is None:
1034 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1035 return s.decode('utf-8').strip()
1037 for path in xmlfiles:
1038 if not os.path.isfile(path):
1040 xml = parse_xml(path)
1041 element = xml.find('string[@name="' + name + '"]')
1042 if element is not None:
1043 content = element_content(element)
1044 return retrieve_string(app_dir, content, xmlfiles)
1049 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1050 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1053 def manifest_paths(app_dir, flavours):
1054 '''Return list of existing files that will be used to find the highest vercode'''
1056 possible_manifests = \
1057 [os.path.join(app_dir, 'AndroidManifest.xml'),
1058 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1059 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1060 os.path.join(app_dir, 'build.gradle')]
1062 for flavour in flavours:
1063 if flavour == 'yes':
1065 possible_manifests.append(
1066 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1068 return [path for path in possible_manifests if os.path.isfile(path)]
1071 def fetch_real_name(app_dir, flavours):
1072 '''Retrieve the package name. Returns the name, or None if not found.'''
1073 for path in manifest_paths(app_dir, flavours):
1074 if not has_extension(path, 'xml') or not os.path.isfile(path):
1076 logging.debug("fetch_real_name: Checking manifest at " + path)
1077 xml = parse_xml(path)
1078 app = xml.find('application')
1081 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1083 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1084 result = retrieve_string_singleline(app_dir, label)
1086 result = result.strip()
1091 def get_library_references(root_dir):
1093 proppath = os.path.join(root_dir, 'project.properties')
1094 if not os.path.isfile(proppath):
1096 with open(proppath, 'r', encoding='iso-8859-1') as f:
1098 if not line.startswith('android.library.reference.'):
1100 path = line.split('=')[1].strip()
1101 relpath = os.path.join(root_dir, path)
1102 if not os.path.isdir(relpath):
1104 logging.debug("Found subproject at %s" % path)
1105 libraries.append(path)
1109 def ant_subprojects(root_dir):
1110 subprojects = get_library_references(root_dir)
1111 for subpath in subprojects:
1112 subrelpath = os.path.join(root_dir, subpath)
1113 for p in get_library_references(subrelpath):
1114 relp = os.path.normpath(os.path.join(subpath, p))
1115 if relp not in subprojects:
1116 subprojects.insert(0, relp)
1120 def remove_debuggable_flags(root_dir):
1121 # Remove forced debuggable flags
1122 logging.debug("Removing debuggable flags from %s" % root_dir)
1123 for root, dirs, files in os.walk(root_dir):
1124 if 'AndroidManifest.xml' in files:
1125 regsub_file(r'android:debuggable="[^"]*"',
1127 os.path.join(root, 'AndroidManifest.xml'))
1130 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1131 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1132 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1135 def app_matches_packagename(app, package):
1138 appid = app.UpdateCheckName or app.id
1139 if appid is None or appid == "Ignore":
1141 return appid == package
1144 def parse_androidmanifests(paths, app):
1146 Extract some information from the AndroidManifest.xml at the given path.
1147 Returns (version, vercode, package), any or all of which might be None.
1148 All values returned are strings.
1151 ignoreversions = app.UpdateCheckIgnore
1152 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1155 return (None, None, None)
1163 if not os.path.isfile(path):
1166 logging.debug("Parsing manifest at {0}".format(path))
1171 if has_extension(path, 'gradle'):
1172 with open(path, 'r') as f:
1174 if gradle_comment.match(line):
1176 # Grab first occurence of each to avoid running into
1177 # alternative flavours and builds.
1179 matches = psearch_g(line)
1181 s = matches.group(2)
1182 if app_matches_packagename(app, s):
1185 matches = vnsearch_g(line)
1187 version = matches.group(2)
1189 matches = vcsearch_g(line)
1191 vercode = matches.group(1)
1194 xml = parse_xml(path)
1195 if "package" in xml.attrib:
1196 s = xml.attrib["package"]
1197 if app_matches_packagename(app, s):
1199 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1200 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1201 base_dir = os.path.dirname(path)
1202 version = retrieve_string_singleline(base_dir, version)
1203 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1204 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1205 if string_is_integer(a):
1208 logging.warning("Problem with xml at {0}".format(path))
1210 # Remember package name, may be defined separately from version+vercode
1212 package = max_package
1214 logging.debug("..got package={0}, version={1}, vercode={2}"
1215 .format(package, version, vercode))
1217 # Always grab the package name and version name in case they are not
1218 # together with the highest version code
1219 if max_package is None and package is not None:
1220 max_package = package
1221 if max_version is None and version is not None:
1222 max_version = version
1224 if vercode is not None \
1225 and (max_vercode is None or vercode > max_vercode):
1226 if not ignoresearch or not ignoresearch(version):
1227 if version is not None:
1228 max_version = version
1229 if vercode is not None:
1230 max_vercode = vercode
1231 if package is not None:
1232 max_package = package
1234 max_version = "Ignore"
1236 if max_version is None:
1237 max_version = "Unknown"
1239 if max_package and not is_valid_package_name(max_package):
1240 raise FDroidException("Invalid package name {0}".format(max_package))
1242 return (max_version, max_vercode, max_package)
1245 def is_valid_package_name(name):
1246 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1249 # Get the specified source library.
1250 # Returns the path to it. Normally this is the path to be used when referencing
1251 # it, which may be a subdirectory of the actual project. If you want the base
1252 # directory of the project, pass 'basepath=True'.
1253 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1254 raw=False, prepare=True, preponly=False, refresh=True,
1263 name, ref = spec.split('@')
1265 number, name = name.split(':', 1)
1267 name, subdir = name.split('/', 1)
1269 if name not in fdroidserver.metadata.srclibs:
1270 raise VCSException('srclib ' + name + ' not found.')
1272 srclib = fdroidserver.metadata.srclibs[name]
1274 sdir = os.path.join(srclib_dir, name)
1277 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1278 vcs.srclib = (name, number, sdir)
1280 vcs.gotorevision(ref, refresh)
1287 libdir = os.path.join(sdir, subdir)
1288 elif srclib["Subdir"]:
1289 for subdir in srclib["Subdir"]:
1290 libdir_candidate = os.path.join(sdir, subdir)
1291 if os.path.exists(libdir_candidate):
1292 libdir = libdir_candidate
1298 remove_signing_keys(sdir)
1299 remove_debuggable_flags(sdir)
1303 if srclib["Prepare"]:
1304 cmd = replace_config_vars(srclib["Prepare"], build)
1306 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1307 if p.returncode != 0:
1308 raise BuildException("Error running prepare command for srclib %s"
1314 return (name, number, libdir)
1317 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1320 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1321 """ Prepare the source code for a particular build
1323 :param vcs: the appropriate vcs object for the application
1324 :param app: the application details from the metadata
1325 :param build: the build details from the metadata
1326 :param build_dir: the path to the build directory, usually 'build/app.id'
1327 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1328 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1330 Returns the (root, srclibpaths) where:
1331 :param root: is the root directory, which may be the same as 'build_dir' or may
1332 be a subdirectory of it.
1333 :param srclibpaths: is information on the srclibs being used
1336 # Optionally, the actual app source can be in a subdirectory
1338 root_dir = os.path.join(build_dir, build.subdir)
1340 root_dir = build_dir
1342 # Get a working copy of the right revision
1343 logging.info("Getting source for revision " + build.commit)
1344 vcs.gotorevision(build.commit, refresh)
1346 # Initialise submodules if required
1347 if build.submodules:
1348 logging.info("Initialising submodules")
1349 vcs.initsubmodules()
1351 # Check that a subdir (if we're using one) exists. This has to happen
1352 # after the checkout, since it might not exist elsewhere
1353 if not os.path.exists(root_dir):
1354 raise BuildException('Missing subdir ' + root_dir)
1356 # Run an init command if one is required
1358 cmd = replace_config_vars(build.init, build)
1359 logging.info("Running 'init' commands in %s" % root_dir)
1361 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1362 if p.returncode != 0:
1363 raise BuildException("Error running init command for %s:%s" %
1364 (app.id, build.versionName), p.output)
1366 # Apply patches if any
1368 logging.info("Applying patches")
1369 for patch in build.patch:
1370 patch = patch.strip()
1371 logging.info("Applying " + patch)
1372 patch_path = os.path.join('metadata', app.id, patch)
1373 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1374 if p.returncode != 0:
1375 raise BuildException("Failed to apply patch %s" % patch_path)
1377 # Get required source libraries
1380 logging.info("Collecting source libraries")
1381 for lib in build.srclibs:
1382 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1383 refresh=refresh, build=build))
1385 for name, number, libpath in srclibpaths:
1386 place_srclib(root_dir, int(number) if number else None, libpath)
1388 basesrclib = vcs.getsrclib()
1389 # If one was used for the main source, add that too.
1391 srclibpaths.append(basesrclib)
1393 # Update the local.properties file
1394 localprops = [os.path.join(build_dir, 'local.properties')]
1396 parts = build.subdir.split(os.sep)
1399 cur = os.path.join(cur, d)
1400 localprops += [os.path.join(cur, 'local.properties')]
1401 for path in localprops:
1403 if os.path.isfile(path):
1404 logging.info("Updating local.properties file at %s" % path)
1405 with open(path, 'r', encoding='iso-8859-1') as f:
1409 logging.info("Creating local.properties file at %s" % path)
1410 # Fix old-fashioned 'sdk-location' by copying
1411 # from sdk.dir, if necessary
1413 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1414 re.S | re.M).group(1)
1415 props += "sdk-location=%s\n" % sdkloc
1417 props += "sdk.dir=%s\n" % config['sdk_path']
1418 props += "sdk-location=%s\n" % config['sdk_path']
1419 ndk_path = build.ndk_path()
1420 # if for any reason the path isn't valid or the directory
1421 # doesn't exist, some versions of Gradle will error with a
1422 # cryptic message (even if the NDK is not even necessary).
1423 # https://gitlab.com/fdroid/fdroidserver/issues/171
1424 if ndk_path and os.path.exists(ndk_path):
1426 props += "ndk.dir=%s\n" % ndk_path
1427 props += "ndk-location=%s\n" % ndk_path
1428 # Add java.encoding if necessary
1430 props += "java.encoding=%s\n" % build.encoding
1431 with open(path, 'w', encoding='iso-8859-1') as f:
1435 if build.build_method() == 'gradle':
1436 flavours = build.gradle
1439 n = build.target.split('-')[1]
1440 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1441 r'compileSdkVersion %s' % n,
1442 os.path.join(root_dir, 'build.gradle'))
1444 # Remove forced debuggable flags
1445 remove_debuggable_flags(root_dir)
1447 # Insert version code and number into the manifest if necessary
1448 if build.forceversion:
1449 logging.info("Changing the version name")
1450 for path in manifest_paths(root_dir, flavours):
1451 if not os.path.isfile(path):
1453 if has_extension(path, 'xml'):
1454 regsub_file(r'android:versionName="[^"]*"',
1455 r'android:versionName="%s"' % build.versionName,
1457 elif has_extension(path, 'gradle'):
1458 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1459 r"""\1versionName '%s'""" % build.versionName,
1462 if build.forcevercode:
1463 logging.info("Changing the version code")
1464 for path in manifest_paths(root_dir, flavours):
1465 if not os.path.isfile(path):
1467 if has_extension(path, 'xml'):
1468 regsub_file(r'android:versionCode="[^"]*"',
1469 r'android:versionCode="%s"' % build.versionCode,
1471 elif has_extension(path, 'gradle'):
1472 regsub_file(r'versionCode[ =]+[0-9]+',
1473 r'versionCode %s' % build.versionCode,
1476 # Delete unwanted files
1478 logging.info("Removing specified files")
1479 for part in getpaths(build_dir, build.rm):
1480 dest = os.path.join(build_dir, part)
1481 logging.info("Removing {0}".format(part))
1482 if os.path.lexists(dest):
1483 if os.path.islink(dest):
1484 FDroidPopen(['unlink', dest], output=False)
1486 FDroidPopen(['rm', '-rf', dest], output=False)
1488 logging.info("...but it didn't exist")
1490 remove_signing_keys(build_dir)
1492 # Add required external libraries
1494 logging.info("Collecting prebuilt libraries")
1495 libsdir = os.path.join(root_dir, 'libs')
1496 if not os.path.exists(libsdir):
1498 for lib in build.extlibs:
1500 logging.info("...installing extlib {0}".format(lib))
1501 libf = os.path.basename(lib)
1502 libsrc = os.path.join(extlib_dir, lib)
1503 if not os.path.exists(libsrc):
1504 raise BuildException("Missing extlib file {0}".format(libsrc))
1505 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1507 # Run a pre-build command if one is required
1509 logging.info("Running 'prebuild' commands in %s" % root_dir)
1511 cmd = replace_config_vars(build.prebuild, build)
1513 # Substitute source library paths into prebuild commands
1514 for name, number, libpath in srclibpaths:
1515 libpath = os.path.relpath(libpath, root_dir)
1516 cmd = cmd.replace('$$' + name + '$$', libpath)
1518 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1519 if p.returncode != 0:
1520 raise BuildException("Error running prebuild command for %s:%s" %
1521 (app.id, build.versionName), p.output)
1523 # Generate (or update) the ant build file, build.xml...
1524 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1525 parms = ['android', 'update', 'lib-project']
1526 lparms = ['android', 'update', 'project']
1529 parms += ['-t', build.target]
1530 lparms += ['-t', build.target]
1531 if build.androidupdate:
1532 update_dirs = build.androidupdate
1534 update_dirs = ant_subprojects(root_dir) + ['.']
1536 for d in update_dirs:
1537 subdir = os.path.join(root_dir, d)
1539 logging.debug("Updating main project")
1540 cmd = parms + ['-p', d]
1542 logging.debug("Updating subproject %s" % d)
1543 cmd = lparms + ['-p', d]
1544 p = SdkToolsPopen(cmd, cwd=root_dir)
1545 # Check to see whether an error was returned without a proper exit
1546 # code (this is the case for the 'no target set or target invalid'
1548 if p.returncode != 0 or p.output.startswith("Error: "):
1549 raise BuildException("Failed to update project at %s" % d, p.output)
1550 # Clean update dirs via ant
1552 logging.info("Cleaning subproject %s" % d)
1553 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1555 return (root_dir, srclibpaths)
1558 # Extend via globbing the paths from a field and return them as a map from
1559 # original path to resulting paths
1560 def getpaths_map(build_dir, globpaths):
1564 full_path = os.path.join(build_dir, p)
1565 full_path = os.path.normpath(full_path)
1566 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1568 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1572 # Extend via globbing the paths from a field and return them as a set
1573 def getpaths(build_dir, globpaths):
1574 paths_map = getpaths_map(build_dir, globpaths)
1576 for k, v in paths_map.items():
1583 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1587 """permanent store of existing APKs with the date they were added
1589 This is currently the only way to permanently store the "updated"
1594 self.path = os.path.join('stats', 'known_apks.txt')
1596 if os.path.isfile(self.path):
1597 with open(self.path, 'r', encoding='utf8') as f:
1599 t = line.rstrip().split(' ')
1601 self.apks[t[0]] = (t[1], None)
1603 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1604 self.changed = False
1606 def writeifchanged(self):
1607 if not self.changed:
1610 if not os.path.exists('stats'):
1614 for apk, app in self.apks.items():
1616 line = apk + ' ' + appid
1618 line += ' ' + added.strftime('%Y-%m-%d')
1621 with open(self.path, 'w', encoding='utf8') as f:
1622 for line in sorted(lst, key=natural_key):
1623 f.write(line + '\n')
1625 def recordapk(self, apkName, app, default_date=None):
1627 Record an apk (if it's new, otherwise does nothing)
1628 Returns the date it was added as a datetime instance
1630 if apkName not in self.apks:
1631 if default_date is None:
1632 default_date = datetime.utcnow()
1633 self.apks[apkName] = (app, default_date)
1635 _, added = self.apks[apkName]
1638 # Look up information - given the 'apkname', returns (app id, date added/None).
1639 # Or returns None for an unknown apk.
1640 def getapp(self, apkname):
1641 if apkname in self.apks:
1642 return self.apks[apkname]
1645 # Get the most recent 'num' apps added to the repo, as a list of package ids
1646 # with the most recent first.
1647 def getlatest(self, num):
1649 for apk, app in self.apks.items():
1653 if apps[appid] > added:
1657 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1658 lst = [app for app, _ in sortedapps]
1663 def get_file_extension(filename):
1664 """get the normalized file extension, can be blank string but never None"""
1665 if isinstance(filename, bytes):
1666 filename = filename.decode('utf-8')
1667 return os.path.splitext(filename)[1].lower()[1:]
1670 def get_apk_debuggable_aapt(apkfile):
1671 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1673 if p.returncode != 0:
1674 raise FDroidException("Failed to get apk manifest information")
1675 for line in p.output.splitlines():
1676 if 'android:debuggable' in line and not line.endswith('0x0'):
1681 def get_apk_debuggable_androguard(apkfile):
1683 from androguard.core.bytecodes.apk import APK
1685 raise FDroidException("androguard library is not installed and aapt not present")
1687 apkobject = APK(apkfile)
1688 if apkobject.is_valid_APK():
1689 debuggable = apkobject.get_element("application", "debuggable")
1690 if debuggable is not None:
1691 return bool(strtobool(debuggable))
1695 def isApkAndDebuggable(apkfile):
1696 """Returns True if the given file is an APK and is debuggable
1698 :param apkfile: full path to the apk to check"""
1700 if get_file_extension(apkfile) != 'apk':
1703 if SdkToolsPopen(['aapt', 'version'], output=False):
1704 return get_apk_debuggable_aapt(apkfile)
1706 return get_apk_debuggable_androguard(apkfile)
1711 self.returncode = None
1715 def SdkToolsPopen(commands, cwd=None, output=True):
1717 if cmd not in config:
1718 config[cmd] = find_sdk_tools_cmd(commands[0])
1719 abscmd = config[cmd]
1721 raise FDroidException("Could not find '%s' on your system" % cmd)
1723 test_aapt_version(config['aapt'])
1724 return FDroidPopen([abscmd] + commands[1:],
1725 cwd=cwd, output=output)
1728 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1730 Run a command and capture the possibly huge output as bytes.
1732 :param commands: command and argument list like in subprocess.Popen
1733 :param cwd: optionally specifies a working directory
1734 :param envs: a optional dictionary of environment variables and their values
1735 :returns: A PopenResult.
1740 set_FDroidPopen_env()
1742 process_env = env.copy()
1743 if envs is not None and len(envs) > 0:
1744 process_env.update(envs)
1747 cwd = os.path.normpath(cwd)
1748 logging.debug("Directory: %s" % cwd)
1749 logging.debug("> %s" % ' '.join(commands))
1751 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1752 result = PopenResult()
1755 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1756 stdout=subprocess.PIPE, stderr=stderr_param)
1757 except OSError as e:
1758 raise BuildException("OSError while trying to execute " +
1759 ' '.join(commands) + ': ' + str(e))
1761 if not stderr_to_stdout and options.verbose:
1762 stderr_queue = Queue()
1763 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1765 while not stderr_reader.eof():
1766 while not stderr_queue.empty():
1767 line = stderr_queue.get()
1768 sys.stderr.buffer.write(line)
1773 stdout_queue = Queue()
1774 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1777 # Check the queue for output (until there is no more to get)
1778 while not stdout_reader.eof():
1779 while not stdout_queue.empty():
1780 line = stdout_queue.get()
1781 if output and options.verbose:
1782 # Output directly to console
1783 sys.stderr.buffer.write(line)
1789 result.returncode = p.wait()
1790 result.output = buf.getvalue()
1795 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1797 Run a command and capture the possibly huge output as a str.
1799 :param commands: command and argument list like in subprocess.Popen
1800 :param cwd: optionally specifies a working directory
1801 :param envs: a optional dictionary of environment variables and their values
1802 :returns: A PopenResult.
1804 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
1805 result.output = result.output.decode('utf-8', 'ignore')
1809 gradle_comment = re.compile(r'[ ]*//')
1810 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1811 gradle_line_matches = [
1812 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1813 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1814 re.compile(r'.*\.readLine\(.*'),
1818 def remove_signing_keys(build_dir):
1819 for root, dirs, files in os.walk(build_dir):
1820 if 'build.gradle' in files:
1821 path = os.path.join(root, 'build.gradle')
1823 with open(path, "r", encoding='utf8') as o:
1824 lines = o.readlines()
1830 with open(path, "w", encoding='utf8') as o:
1831 while i < len(lines):
1834 while line.endswith('\\\n'):
1835 line = line.rstrip('\\\n') + lines[i]
1838 if gradle_comment.match(line):
1843 opened += line.count('{')
1844 opened -= line.count('}')
1847 if gradle_signing_configs.match(line):
1852 if any(s.match(line) for s in gradle_line_matches):
1860 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1863 'project.properties',
1865 'default.properties',
1866 'ant.properties', ]:
1867 if propfile in files:
1868 path = os.path.join(root, propfile)
1870 with open(path, "r", encoding='iso-8859-1') as o:
1871 lines = o.readlines()
1875 with open(path, "w", encoding='iso-8859-1') as o:
1877 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1884 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1887 def set_FDroidPopen_env(build=None):
1889 set up the environment variables for the build environment
1891 There is only a weak standard, the variables used by gradle, so also set
1892 up the most commonly used environment variables for SDK and NDK. Also, if
1893 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1895 global env, orig_path
1899 orig_path = env['PATH']
1900 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1901 env[n] = config['sdk_path']
1902 for k, v in config['java_paths'].items():
1903 env['JAVA%s_HOME' % k] = v
1905 missinglocale = True
1906 for k, v in env.items():
1907 if k == 'LANG' and v != 'C':
1908 missinglocale = False
1910 missinglocale = False
1912 env['LANG'] = 'en_US.UTF-8'
1914 if build is not None:
1915 path = build.ndk_path()
1916 paths = orig_path.split(os.pathsep)
1917 if path not in paths:
1918 paths = [path] + paths
1919 env['PATH'] = os.pathsep.join(paths)
1920 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1921 env[n] = build.ndk_path()
1924 def replace_build_vars(cmd, build):
1925 cmd = cmd.replace('$$COMMIT$$', build.commit)
1926 cmd = cmd.replace('$$VERSION$$', build.versionName)
1927 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1931 def replace_config_vars(cmd, build):
1932 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1933 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1934 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1935 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1936 if build is not None:
1937 cmd = replace_build_vars(cmd, build)
1941 def place_srclib(root_dir, number, libpath):
1944 relpath = os.path.relpath(libpath, root_dir)
1945 proppath = os.path.join(root_dir, 'project.properties')
1948 if os.path.isfile(proppath):
1949 with open(proppath, "r", encoding='iso-8859-1') as o:
1950 lines = o.readlines()
1952 with open(proppath, "w", encoding='iso-8859-1') as o:
1955 if line.startswith('android.library.reference.%d=' % number):
1956 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1961 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1964 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1967 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1968 """Verify that two apks are the same
1970 One of the inputs is signed, the other is unsigned. The signature metadata
1971 is transferred from the signed to the unsigned apk, and then jarsigner is
1972 used to verify that the signature from the signed apk is also varlid for
1973 the unsigned one. If the APK given as unsigned actually does have a
1974 signature, it will be stripped out and ignored.
1976 There are two SHA1 git commit IDs that fdroidserver includes in the builds
1977 it makes: fdroidserverid and buildserverid. Originally, these were inserted
1978 into AndroidManifest.xml, but that makes the build not reproducible. So
1979 instead they are included as separate files in the APK's META-INF/ folder.
1980 If those files exist in the signed APK, they will be part of the signature
1981 and need to also be included in the unsigned APK for it to validate.
1983 :param signed_apk: Path to a signed apk file
1984 :param unsigned_apk: Path to an unsigned apk file expected to match it
1985 :param tmp_dir: Path to directory for temporary files
1986 :returns: None if the verification is successful, otherwise a string
1987 describing what went wrong.
1990 signed = ZipFile(signed_apk, 'r')
1991 meta_inf_files = ['META-INF/MANIFEST.MF']
1992 for f in signed.namelist():
1993 if apk_sigfile.match(f) \
1994 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
1995 meta_inf_files.append(f)
1996 if len(meta_inf_files) < 3:
1997 return "Signature files missing from {0}".format(signed_apk)
1999 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2000 unsigned = ZipFile(unsigned_apk, 'r')
2001 # only read the signature from the signed APK, everything else from unsigned
2002 with ZipFile(tmp_apk, 'w') as tmp:
2003 for filename in meta_inf_files:
2004 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2005 for info in unsigned.infolist():
2006 if info.filename in meta_inf_files:
2007 logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
2009 if info.filename in tmp.namelist():
2010 return "duplicate filename found: " + info.filename
2011 tmp.writestr(info, unsigned.read(info.filename))
2015 verified = verify_apk_signature(tmp_apk)
2018 logging.info("...NOT verified - {0}".format(tmp_apk))
2019 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2020 os.path.dirname(unsigned_apk))
2022 logging.info("...successfully verified")
2026 def verify_apk_signature(apk, jar=False):
2027 """verify the signature on an APK
2029 Try to use apksigner whenever possible since jarsigner is very
2030 shitty: unsigned APKs pass as "verified"! So this has to turn on
2031 -strict then check for result 4.
2033 You can set :param: jar to True if you want to use this method
2034 to verify jar signatures.
2036 if set_command_in_config('apksigner'):
2037 args = [config['apksigner'], 'verify']
2039 args += ['--min-sdk-version=1']
2040 return subprocess.call(args + [apk]) == 0
2042 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2043 return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
2046 def verify_old_apk_signature(apk):
2047 """verify the signature on an archived APK, supporting deprecated algorithms
2049 F-Droid aims to keep every single binary that it ever published. Therefore,
2050 it needs to be able to verify APK signatures that include deprecated/removed
2051 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2053 jarsigner passes unsigned APKs as "verified"! So this has to turn
2054 on -strict then check for result 4.
2058 _java_security = os.path.join(os.getcwd(), '.java.security')
2059 with open(_java_security, 'w') as fp:
2060 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2062 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2063 '-strict', '-verify', apk]) == 4
2066 apk_badchars = re.compile('''[/ :;'"]''')
2069 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2072 Returns None if the apk content is the same (apart from the signing key),
2073 otherwise a string describing what's different, or what went wrong when
2074 trying to do the comparison.
2080 absapk1 = os.path.abspath(apk1)
2081 absapk2 = os.path.abspath(apk2)
2083 if set_command_in_config('diffoscope'):
2084 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2085 htmlfile = logfilename + '.diffoscope.html'
2086 textfile = logfilename + '.diffoscope.txt'
2087 if subprocess.call([config['diffoscope'],
2088 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2089 '--html', htmlfile, '--text', textfile,
2090 absapk1, absapk2]) != 0:
2091 return("Failed to unpack " + apk1)
2093 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2094 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2095 for d in [apk1dir, apk2dir]:
2096 if os.path.exists(d):
2099 os.mkdir(os.path.join(d, 'jar-xf'))
2101 if subprocess.call(['jar', 'xf',
2102 os.path.abspath(apk1)],
2103 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2104 return("Failed to unpack " + apk1)
2105 if subprocess.call(['jar', 'xf',
2106 os.path.abspath(apk2)],
2107 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2108 return("Failed to unpack " + apk2)
2110 if set_command_in_config('apktool'):
2111 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2113 return("Failed to unpack " + apk1)
2114 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2116 return("Failed to unpack " + apk2)
2118 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2119 lines = p.output.splitlines()
2120 if len(lines) != 1 or 'META-INF' not in lines[0]:
2121 if set_command_in_config('meld'):
2122 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2123 return("Unexpected diff output - " + p.output)
2125 # since everything verifies, delete the comparison to keep cruft down
2126 shutil.rmtree(apk1dir)
2127 shutil.rmtree(apk2dir)
2129 # If we get here, it seems like they're the same!
2133 def set_command_in_config(command):
2134 '''Try to find specified command in the path, if it hasn't been
2135 manually set in config.py. If found, it is added to the config
2136 dict. The return value says whether the command is available.
2139 if command in config:
2142 tmp = find_command(command)
2144 config[command] = tmp
2149 def find_command(command):
2150 '''find the full path of a command, or None if it can't be found in the PATH'''
2153 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2155 fpath, fname = os.path.split(command)
2160 for path in os.environ["PATH"].split(os.pathsep):
2161 path = path.strip('"')
2162 exe_file = os.path.join(path, command)
2163 if is_exe(exe_file):
2170 '''generate a random password for when generating keys'''
2171 h = hashlib.sha256()
2172 h.update(os.urandom(16)) # salt
2173 h.update(socket.getfqdn().encode('utf-8'))
2174 passwd = base64.b64encode(h.digest()).strip()
2175 return passwd.decode('utf-8')
2178 def genkeystore(localconfig):
2180 Generate a new key with password provided in :param localconfig and add it to new keystore
2181 :return: hexed public key, public key fingerprint
2183 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2184 keystoredir = os.path.dirname(localconfig['keystore'])
2185 if keystoredir is None or keystoredir == '':
2186 keystoredir = os.path.join(os.getcwd(), keystoredir)
2187 if not os.path.exists(keystoredir):
2188 os.makedirs(keystoredir, mode=0o700)
2191 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2192 'FDROID_KEY_PASS': localconfig['keypass'],
2194 p = FDroidPopen([config['keytool'], '-genkey',
2195 '-keystore', localconfig['keystore'],
2196 '-alias', localconfig['repo_keyalias'],
2197 '-keyalg', 'RSA', '-keysize', '4096',
2198 '-sigalg', 'SHA256withRSA',
2199 '-validity', '10000',
2200 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2201 '-keypass:env', 'FDROID_KEY_PASS',
2202 '-dname', localconfig['keydname']], envs=env_vars)
2203 if p.returncode != 0:
2204 raise BuildException("Failed to generate key", p.output)
2205 os.chmod(localconfig['keystore'], 0o0600)
2206 if not options.quiet:
2207 # now show the lovely key that was just generated
2208 p = FDroidPopen([config['keytool'], '-list', '-v',
2209 '-keystore', localconfig['keystore'],
2210 '-alias', localconfig['repo_keyalias'],
2211 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2212 logging.info(p.output.strip() + '\n\n')
2213 # get the public key
2214 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2215 '-keystore', localconfig['keystore'],
2216 '-alias', localconfig['repo_keyalias'],
2217 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2218 + config['smartcardoptions'],
2219 envs=env_vars, output=False, stderr_to_stdout=False)
2220 if p.returncode != 0 or len(p.output) < 20:
2221 raise BuildException("Failed to get public key", p.output)
2223 fingerprint = get_cert_fingerprint(pubkey)
2224 return hexlify(pubkey), fingerprint
2227 def get_cert_fingerprint(pubkey):
2229 Generate a certificate fingerprint the same way keytool does it
2230 (but with slightly different formatting)
2232 digest = hashlib.sha256(pubkey).digest()
2233 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2234 return " ".join(ret)
2237 def get_certificate(certificate_file):
2239 Extracts a certificate from the given file.
2240 :param certificate_file: file bytes (as string) representing the certificate
2241 :return: A binary representation of the certificate's public key, or None in case of error
2243 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2244 if content.getComponentByName('contentType') != rfc2315.signedData:
2246 content = decoder.decode(content.getComponentByName('content'),
2247 asn1Spec=rfc2315.SignedData())[0]
2249 certificates = content.getComponentByName('certificates')
2250 cert = certificates[0].getComponentByName('certificate')
2252 logging.error("Certificates not found.")
2254 return encoder.encode(cert)
2257 def write_to_config(thisconfig, key, value=None, config_file=None):
2258 '''write a key/value to the local config.py
2260 NOTE: only supports writing string variables.
2262 :param thisconfig: config dictionary
2263 :param key: variable name in config.py to be overwritten/added
2264 :param value: optional value to be written, instead of fetched
2265 from 'thisconfig' dictionary.
2268 origkey = key + '_orig'
2269 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2270 cfg = config_file if config_file else 'config.py'
2273 with open(cfg, 'r', encoding="utf-8") as f:
2274 lines = f.readlines()
2276 # make sure the file ends with a carraige return
2278 if not lines[-1].endswith('\n'):
2281 # regex for finding and replacing python string variable
2282 # definitions/initializations
2283 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2284 repl = key + ' = "' + value + '"'
2285 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2286 repl2 = key + " = '" + value + "'"
2288 # If we replaced this line once, we make sure won't be a
2289 # second instance of this line for this key in the document.
2292 with open(cfg, 'w', encoding="utf-8") as f:
2294 if pattern.match(line) or pattern2.match(line):
2296 line = pattern.sub(repl, line)
2297 line = pattern2.sub(repl2, line)
2308 def parse_xml(path):
2309 return XMLElementTree.parse(path).getroot()
2312 def string_is_integer(string):
2320 def get_per_app_repos():
2321 '''per-app repos are dirs named with the packageName of a single app'''
2323 # Android packageNames are Java packages, they may contain uppercase or
2324 # lowercase letters ('A' through 'Z'), numbers, and underscores
2325 # ('_'). However, individual package name parts may only start with
2326 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2327 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2330 for root, dirs, files in os.walk(os.getcwd()):
2332 print('checking', root, 'for', d)
2333 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2334 # standard parts of an fdroid repo, so never packageNames
2337 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2343 def is_repo_file(filename):
2344 '''Whether the file in a repo is a build product to be delivered to users'''
2345 if isinstance(filename, str):
2346 filename = filename.encode('utf-8', errors="surrogateescape")
2347 return os.path.isfile(filename) \
2348 and not filename.endswith(b'.asc') \
2349 and not filename.endswith(b'.sig') \
2350 and os.path.basename(filename) not in [
2352 b'index_unsigned.jar',