import xml.etree.ElementTree as XMLElementTree
from binascii import hexlify
-from datetime import datetime
+from datetime import datetime, timedelta
from distutils.version import LooseVersion
from queue import Queue
from zipfile import ZipFile
'r13b': None,
'r14b': None,
'r15c': None,
+ 'r16': None,
},
'qt_sdk_path': None,
'build_tools': "25.0.2",
return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
-def read_pkg_args(args, allow_vercodes=False):
+def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
"""
- :param args: arguments in the form of multiple appid:[vc] strings
+ :param appids: arguments in the form of multiple appid:[vc] strings
:returns: a dictionary with the set of vercodes specified for each package
"""
-
vercodes = {}
- if not args:
+ if not appid_versionCode_pairs:
return vercodes
- for p in args:
+ for p in appid_versionCode_pairs:
if allow_vercodes and ':' in p:
package, vercode = p.split(':')
else:
return vercodes
-def read_app_args(args, allapps, allow_vercodes=False):
- """
- On top of what read_pkg_args does, this returns the whole app metadata, but
- limiting the builds list to the builds matching the vercodes specified.
+def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
+ """Build a list of App instances for processing
+
+ On top of what read_pkg_args does, this returns the whole app
+ metadata, but limiting the builds list to the builds matching the
+ appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
+ all App and Build instances are returned.
+
"""
- vercodes = read_pkg_args(args, allow_vercodes)
+ vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
if not vercodes:
return allapps
else:
remote = app.Repo
vcs = getvcs(app.RepoType, remote, build_dir)
- logging.debug("Using %s" % vcs.clientversion())
return vcs, build_dir
def clientversioncmd(self):
return ['git', '--version']
+ def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
+ '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
+
+ While fetch/pull/clone respect the command line option flags,
+ it seems that submodule commands do not. They do seem to
+ follow whatever is in env vars, if the version of git is new
+ enough. So we just throw the kitchen sink at it to see what
+ sticks.
+
+ '''
+ if cwd is None:
+ cwd = self.local
+ git_config = []
+ for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+ git_config.append('-c')
+ git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
+ git_config.append('-c')
+ git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
+ git_config.append('-c')
+ git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
+ # add helpful tricks supported in git >= 2.3
+ ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes'
+ git_config.append('-c')
+ git_config.append('core.sshCommand="' + ssh_command + '"') # git >= 2.10
+ envs.update({
+ 'GIT_TERMINAL_PROMPT': '0',
+ 'GIT_SSH_COMMAND': ssh_command, # git >= 2.3
+ })
+ return FDroidPopen(['git', ] + git_config + gitargs,
+ envs=envs, cwd=cwd, output=output)
+
def checkrepo(self):
"""If the local directory exists, but is somehow not a git repository,
git will traverse up the directory tree until it finds one
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
# Brand new checkout
- p = FDroidPopen(['git', 'clone', self.remote, self.local])
+ p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git clone failed", p.output)
raise VCSException(_("Git clean failed"), p.output)
if not self.refreshed:
# Get latest commits and tags from remote
- p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
+ p = self.GitFetchFDroidPopen(['fetch', 'origin'])
if p.returncode != 0:
raise VCSException(_("Git fetch failed"), p.output)
- p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
+ p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False)
if p.returncode != 0:
raise VCSException(_("Git fetch failed"), p.output)
# Recreate origin/HEAD as git clone would do it, in case it disappeared
lines = f.readlines()
with open(submfile, 'w') as f:
for line in lines:
- if 'git@github.com' in line:
- line = line.replace('git@github.com:', 'https://github.com/')
- if 'git@gitlab.com' in line:
- line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
+ for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+ line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
f.write(line)
p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException(_("Git submodule sync failed"), p.output)
- p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
+ p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'])
if p.returncode != 0:
raise VCSException(_("Git submodule update failed"), p.output)
vercode = None
package = None
+ flavour = ""
+ if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
+ flavour = app.builds[-1].gradle[-1]
+
if has_extension(path, 'gradle'):
+ # first try to get version name and code from correct flavour
with open(path, 'r') as f:
- for line in f:
- if gradle_comment.match(line):
- continue
- # Grab first occurence of each to avoid running into
- # alternative flavours and builds.
- if not package:
- matches = psearch_g(line)
- if matches:
- s = matches.group(2)
- if app_matches_packagename(app, s):
- package = s
- if not version:
- matches = vnsearch_g(line)
- if matches:
- version = matches.group(2)
- if not vercode:
- matches = vcsearch_g(line)
- if matches:
- vercode = matches.group(1)
+ buildfile = f.read()
+
+ regex_string = r"" + flavour + ".*?}"
+ search = re.compile(regex_string, re.DOTALL)
+ result = search.search(buildfile)
+
+ if result is not None:
+ resultgroup = result.group()
+
+ if not package:
+ matches = psearch_g(resultgroup)
+ if matches:
+ s = matches.group(2)
+ if app_matches_packagename(app, s):
+ package = s
+ if not version:
+ matches = vnsearch_g(resultgroup)
+ if matches:
+ version = matches.group(2)
+ if not vercode:
+ matches = vcsearch_g(resultgroup)
+ if matches:
+ vercode = matches.group(1)
+ else:
+ # fall back to parse file line by line
+ with open(path, 'r') as f:
+ for line in f:
+ if gradle_comment.match(line):
+ continue
+ # Grab first occurence of each to avoid running into
+ # alternative flavours and builds.
+ if not package:
+ matches = psearch_g(line)
+ if matches:
+ s = matches.group(2)
+ if app_matches_packagename(app, s):
+ package = s
+ if not version:
+ matches = vnsearch_g(line)
+ if matches:
+ version = matches.group(2)
+ if not vercode:
+ matches = vcsearch_g(line)
+ if matches:
+ vercode = matches.group(1)
else:
try:
xml = parse_xml(path)
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
- if os.path.islink(dest):
- FDroidPopen(['unlink', dest], output=False)
+ # rmtree can only handle directories that are not symlinks, so catch anything else
+ if not os.path.isdir(dest) or os.path.islink(dest):
+ os.remove(dest)
else:
- FDroidPopen(['rm', '-rf', dest], output=False)
+ shutil.rmtree(dest)
else:
logging.info("...but it didn't exist")
return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
+def check_system_clock(dt_obj, path):
+ """Check if system clock is updated based on provided date
+
+ If an APK has files newer than the system time, suggest updating
+ the system clock. This is useful for offline systems, used for
+ signing, which do not have another source of clock sync info. It
+ has to be more than 24 hours newer because ZIP/APK files do not
+ store timezone info
+
+ """
+ checkdt = dt_obj - timedelta(1)
+ if datetime.today() < checkdt:
+ logging.warning(_('System clock is older than date in {path}!').format(path=path)
+ + '\n' + _('Set clock to that time using:') + '\n'
+ + 'sudo date -s "' + str(dt_obj) + '"')
+
+
class KnownApks:
"""permanent store of existing APKs with the date they were added
date = datetime.strptime(t[-1], '%Y-%m-%d')
filename = line[0:line.rfind(appid) - 1]
self.apks[filename] = (appid, date)
+ check_system_clock(date, self.path)
self.changed = False
def writeifchanged(self):
return False
+def local_rsync(options, fromdir, todir):
+ '''Rsync method for local to local copying of things
+
+ This is an rsync wrapper with all the settings for safe use within
+ the various fdroidserver use cases. This uses stricter rsync
+ checking on all files since people using offline mode are already
+ prioritizing security above ease and speed.
+
+ '''
+ rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
+ '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
+ if not options.no_checksum:
+ rsyncargs.append('--checksum')
+ if options.verbose:
+ rsyncargs += ['--verbose']
+ if options.quiet:
+ rsyncargs += ['--quiet']
+ logging.debug(' '.join(rsyncargs + [fromdir, todir]))
+ if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
+ raise FDroidException()
+
+
def get_per_app_repos():
'''per-app repos are dirs named with the packageName of a single app'''
b'index-v1.json',
b'categories.txt',
]
+
+
+def get_examples_dir():
+ '''Return the dir where the fdroidserver example files are available'''
+ examplesdir = None
+ tmp = os.path.dirname(sys.argv[0])
+ if os.path.basename(tmp) == 'bin':
+ egg_links = glob.glob(os.path.join(tmp, '..',
+ 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
+ if egg_links:
+ # installed from local git repo
+ examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
+ else:
+ # try .egg layout
+ examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
+ if not os.path.exists(examplesdir): # use UNIX layout
+ examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
+ else:
+ # we're running straight out of the git repo
+ prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+ examplesdir = prefix + '/examples'
+
+ return examplesdir
~ianmdlvl / fdroidserver.git / blobdiff