*.pyc
*.class
*.box
+
# files generated by build
build/
dist/
fdroidserver.egg-info/
pylint.parseable
/.testfiles/
+docs/html/
+
+# files generated by tests
+tests/getsig/tmp/
-include README
+include README.md
include COPYING
include fd-commit
include fdroid
include jenkins-build
include makebuildserver
-include updateplugin
include buildserver/config.buildserver.py
include buildserver/fixpaths.sh
include buildserver/cookbooks/android-ndk/recipes/default.rb
include examples/fdroid-icon.png
include examples/makebs.config.py
include examples/opensc-fdroid.cfg
-include fdroidserver/getsig/run.sh
-include fdroidserver/getsig/make.sh
-include fdroidserver/getsig/getsig.java
+include tests/getsig/run.sh
+include tests/getsig/make.sh
+include tests/getsig/getsig.java
include tests/run-tests
+include tests/update.TestCase
include tests/urzip.apk
+include tests/urzip-badsig.apk
include wp-fdroid/AndroidManifest.xml
include wp-fdroid/android-permissions.php
include wp-fdroid/readme.txt
+++ /dev/null
-F-Droid is an installable catalogue of FOSS (Free and Open Source Software)
-applications for the Android platform. The client makes it easy to browse,
-install, and keep track of updates on your device.
-
-The F-Droid server tools provide various scripts and tools that are used to
-maintain the main F-Droid application repository. You can use these same tools
-to create your own additional or alternative repository for publishing, or to
-assist in creating, testing and submitting metadata to the main repository.
-
-For documentation, please see the docs directory.
-
-Alternatively, visit https://f-droid.org/manual/
-
-
-Installing
-----------
-
-The easiest way to install the fdroidserver tools is to use virtualenv and pip
-(if you are Debian/Ubuntu/Mint/etc, you can first try installing using
-`apt-get install fdroidserver`). First, make sure you have virtualenv
-installed, it should be included in your OS's Python distribution or via other
-mechanisms like dnf/yum/pacman/emerge/Fink/MacPorts/Brew. Then here's how to
-install:
-
- git clone https://gitlab.com/fdroid/fdroidserver.git
- cd fdroidserver
- virtualenv env/
- . env/bin/activate
- pip install -e .
- python2 setup.py install
--- /dev/null
+F-Droid Server
+==============
+
+Server for [F-Droid](https://f-droid.org), the Free Software repository system
+for Android.
+
+The F-Droid server tools provide various scripts and tools that are used to
+maintain the main [F-Droid application repository](https://f-droid.org/repository/browse).
+You can use these same tools to create your own additional or alternative
+repository for publishing, or to assist in creating, testing and submitting
+metadata to the main repository.
+
+For documentation, please see the docs directory.
+
+Alternatively, visit [https://f-droid.org/manual/](https://f-droid.org/manual/).
+
+What is F-Droid?
+----------------
+
+F-Droid is an installable catalogue of FOSS (Free and Open Source Software)
+applications for the Android platform. The client makes it easy to browse,
+install, and keep track of updates on your device.
+
+Installing
+----------
+
+The easiest way to install the `fdroidserver` tools is on Ubuntu, Mint or other
+Ubuntu based distributions, you can install using:
+
+```
+sudo apt-get install fdroidserver
+```
+
+For older Ubuntu releases or to get the latest version, you can get
+`fdroidserver` from the Guardian Project PPA (the signing key
+fingerprint is `6B80 A842 07B3 0AC9 DEE2 35FE F50E ADDD 2234 F563`)
+
+```
+sudo add-apt-repository ppa:guardianproject/ppa
+sudo apt-get update
+sudo apt-get install fdroidserver
+```
+
+On OSX, `fdroidserver` is available from third party package managers,
+like Homebrew, MacPorts, and Fink:
+
+```
+sudo brew install fdroidserver
+```
+
+For Arch-Linux is a package in the AUR available. If you have installed
+`yaourt` or something similiar, you can do:
+
+```
+yaourt -S fdroidserver
+```
+
+For any platform where Python's `easy_install` is an option (e.g. OSX
+or Cygwin, you can use it:
+
+```
+sudo easy_install fdroidserver
+```
+
+Python's `pip` also works:
+
+```
+sudo pip install fdroidserver
+```
+
+The combination of `virtualenv` and `pip` is great for testing out the
+latest versions of `fdroidserver`. Using `pip`, `fdroidserver` can
+even be installed straight from git. First, make sure you have
+installed the python header files, virtualenv and pip. They should be
+included in your OS's default package manager or you can install them
+via other mechanisms like Brew/dnf/pacman/emerge/Fink/MacPorts.
+
+For Debian based distributions:
+
+```
+apt-get install python-dev python-pip python-virtualenv
+```
+Then here's how to install:
+
+```
+git clone https://gitlab.com/fdroid/fdroidserver.git
+cd fdroidserver
+virtualenv env/
+source env/bin/activate
+pip install -e .
+python2 setup.py install
+```
sdk_path = "/home/vagrant/android-sdk"
-ndk_path = "/home/vagrant/android-ndk"
-build_tools = "20.0.0"
-ant = "ant"
-mvn3 = "mvn"
-gradle = "gradle"
+ndk_paths = {
+ 'r9b': "/home/vagrant/android-ndk/r9b",
+ 'r10e': "/home/vagrant/android-ndk/r10e",
+}
ndk_loc = node[:settings][:ndk_loc]
user = node[:settings][:user]
-execute "add-android-ndk-path" do
- user user
- command "echo \"export PATH=\\$PATH:#{ndk_loc} #PATH-NDK\" >> /home/#{user}/.bsenv"
- not_if "grep PATH-NDK /home/#{user}/.bsenv"
+script "setup-android-ndk" do
+ timeout 14400
+ interpreter "bash"
+ user node[:settings][:user]
+ cwd "/tmp"
+ code "
+ mkdir #{ndk_loc}
+ "
+ not_if do
+ File.exists?("#{ndk_loc}")
+ end
end
-script "setup-android-ndk" do
+script "setup-android-ndk-r9b" do
timeout 14400
interpreter "bash"
user node[:settings][:user]
fi
tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
- mv android-ndk-r9b #{ndk_loc}
+ mv android-ndk-r9b #{ndk_loc}/r9b
"
not_if do
- File.exists?("#{ndk_loc}")
+ File.exists?("#{ndk_loc}/r9b")
+ end
+end
+
+script "setup-android-ndk-r10e" do
+ timeout 14400
+ interpreter "bash"
+ user node[:settings][:user]
+ cwd "/tmp"
+ code "
+ if [ `uname -m` == 'x86_64' ] ; then
+ SUFFIX='_64'
+ else
+ SUFFIX=''
+ fi
+ chmod u+x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin
+ /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin x
+ mv android-ndk-r10e #{ndk_loc}/r10e
+ "
+ not_if do
+ File.exists?("#{ndk_loc}/r10e")
end
end
user user
cwd "/tmp"
code "
- tar zxvf /vagrant/cache/android-sdk_r23.0.2-linux.tgz
+ tar zxvf /vagrant/cache/android-sdk_r24.3.4-linux.tgz
mv android-sdk-linux #{sdk_loc}
#{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
#{sdk_loc}/tools/android update sdk --no-ui -t tool
script "add_build_tools" do
interpreter "bash"
user user
- ver = "20.0.0"
+ ver = "23.0.0"
cwd "/tmp"
code "
if [ -f /vagrant/cache/build-tools/#{ver}.tar.gz ] ; then
%w{android-3 android-4 android-5 android-6 android-7 android-8 android-9
android-10 android-11 android-12 android-13 android-14 android-15
- android-16 android-17 android-18 android-19 android-20
+ android-16 android-17 android-18 android-19 android-20 android-21
+ android-22 android-23
extra-android-support extra-android-m2repository}.each do |sdk|
script "add_sdk_#{sdk}" do
command "apt-get update"
end
-%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip yasm imagemagick gettext realpath transfig texinfo curl librsvg2-bin xsltproc vorbis-tools swig quilt faketime optipng python-gnupg}.each do |pkg|
+%w{ant ant-contrib autoconf autoconf2.13 automake1.11 autopoint bison bzr cmake curl expect faketime flex gettext git-core git-svn gperf graphviz imagemagick inkscape javacc libarchive-zip-perl librsvg2-bin libsaxonb-java libssl-dev libssl1.0.0 libtool make maven mercurial nasm openjdk-7-jdk optipng pandoc perlmagick pkg-config python python-gnupg python-magic python-setuptools python3-gnupg quilt realpath scons subversion swig texinfo transfig unzip vorbis-tools xsltproc yasm zip}.each do |pkg|
package pkg do
action :install
end
end
end
+easy_install_package "compare-locales" do
+ options "-U"
+ action :install
+end
+
execute "add-bsenv" do
user user
command "echo \". ./.bsenv \" >> /home/#{user}/.bashrc"
not_if "test -d /opt/gradle/versions"
end
-%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12}.each do |ver|
+%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12 2.1 2.2.1 2.3 2.4 2.5 2.6}.each do |ver|
script "install-gradle-#{ver}" do
cwd "/tmp"
interpreter "bash"
verdir="${basedir}/versions"
args=("$@")
-v_all=($(cd ${verdir} && ls | sort -rV))
+v_all=($(cd "${verdir}" && ls | sort -rV))
echo "Available gradle versions: ${v_all[@]}"
run_gradle() {
- ${verdir}/${v_found}/bin/gradle "${args[@]}"
+ "${verdir}/${v_found}/bin/gradle" "${args[@]}"
exit $?
}
# key-value pairs of what gradle version each gradle plugin version
# should accept
-d_plugin_k=(0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2)
-d_plugin_v=(1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
+d_plugin_k=(1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2)
+d_plugin_v=(2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
-for v in ${d_plugin_v}; do
- contains $v "${v_all[*]}" && v_def=$v && break
+# All gradle versions we know about
+plugin_v=(2.6 2.5 2.4 2.3 2.2.1 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4)
+
+# Find the highest version available
+for v in ${plugin_v}; do
+ if contains $v "${v_all[*]}"; then
+ v_def=$v
+ break
+ fi
done
-# Latest takes priority
-for f in ../build.gradle build.gradle; do
+# Earliest takes priority
+for f in build.gradle ../build.gradle; do
[[ -f $f ]] || continue
while read l; do
if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then
plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/")
elif [[ -z "$wrapper_ver" && $l == *'gradleVersion'* ]]; then
- wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion[ ]*=[ ]*[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/")
+ wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/")
fi
done < $f
done
__complete_options() {
case "${cur}" in
--*)
- COMPREPLY=( $( compgen -W "${lopts}" -- $cur ) )
+ COMPREPLY=( $( compgen -W "--help ${lopts}" -- $cur ) )
return 0;;
*)
- COMPREPLY=( $( compgen -W "${opts} ${lopts}" -- $cur ) )
+ COMPREPLY=( $( compgen -W "-h ${opts} --help ${lopts}" -- $cur ) )
return 0;;
esac
}
__complete_build() {
- opts="-h -v -q -l -s -t -f -a -w"
+ opts="-v -q -l -s -t -f -a -w"
- lopts="--help --verbose --quiet --latest --stop --test --server --resetserver
- --on-server --skip-scan --no-tarball --force --all --wiki"
+ lopts="--verbose --quiet --latest --stop --test --server --resetserver
+ --on-server --skip-scan --no-tarball --force --all --wiki --no-refresh"
case "${cur}" in
-*)
__complete_options
}
__complete_install() {
- opts="-h -v -q"
- lopts="--help --verbose --quiet --all"
+ opts="-v -q"
+ lopts="--verbose --quiet --all"
case "${cur}" in
-*)
__complete_options
}
__complete_update() {
- opts="-h -c -v -q -b -i -I -e -w"
- lopts="--help --create-metadata --verbose --quiet --buildreport
- --interactive --icons --editor --wiki --pretty --clean --delete-unknown"
+ opts="-c -v -q -b -i -I -e -w"
+ lopts="--create-metadata --verbose --quiet --buildreport
+ --interactive --icons --editor --wiki --pretty --clean --delete-unknown
+ --nosign"
case "${prev}" in
-e|--editor)
_filedir
}
__complete_publish() {
- opts="-h -v -q"
- lopts="--help --verbose --quiet"
+ opts="-v -q"
+ lopts="--verbose --quiet"
case "${cur}" in
-*)
__complete_options
}
__complete_checkupdates() {
- opts="-h -v -q"
- lopts="--help --verbose --quiet --auto --autoonly --commit --gplay"
+ opts="-v -q"
+ lopts="--verbose --quiet --auto --autoonly --commit --gplay"
case "${cur}" in
-*)
__complete_options
}
__complete_import() {
- opts="-h -u -s -r -q"
- lopts="--help --url --subdir --repo --rev --quiet"
+ opts="-u -s -q"
+ lopts="--url --subdir --rev --quiet"
case "${prev}" in
- -u|--url|-r|--repo|-s|--subdir|--rev) return 0;;
+ -u|--url|-s|--subdir|--rev) return 0;;
esac
__complete_options
}
__complete_readmeta() {
- opts="-h -v -q"
- lopts="--help --verbose --quiet"
+ opts="-v -q"
+ lopts="--verbose --quiet"
__complete_options
}
__complete_rewritemeta() {
- opts="-h -v -q"
- lopts="--help --verbose --quiet"
+ opts="-v -q"
+ lopts="--verbose --quiet"
case "${cur}" in
-*)
__complete_options
}
__complete_lint() {
- opts="-h -v -q -p"
- lopts="--help --verbose --quiet --pedantic"
+ opts="-v -q"
+ lopts="--verbose --quiet"
case "${cur}" in
-*)
__complete_options
}
__complete_scanner() {
- opts="-h -v -q"
- lopts="--help --verbose --quiet --nosvn"
+ opts="-v -q"
+ lopts="--verbose --quiet"
case "${cur}" in
-*)
__complete_options
}
__complete_verify() {
- opts="-h -v -q -p"
- lopts="--help --verbose --quiet"
+ opts="-v -q -p"
+ lopts="--verbose --quiet"
case "${cur}" in
-*)
__complete_options
}
__complete_stats() {
- opts="-h -v -q -d"
- lopts="--help --verbose --quiet --download"
+ opts="-v -q -d"
+ lopts="--verbose --quiet --download"
__complete_options
}
__complete_server() {
- opts="-h -i -v -q"
- lopts="--help --identity-file --verbose --quiet update"
+ opts="-i -v -q"
+ lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
+ --verbose --quiet --no-checksum update"
+ __complete_options
+}
+
+__complete_signindex() {
+ opts="-v -q"
+ lopts="--verbose"
__complete_options
}
__complete_init() {
- opts="-h -v -q -d"
- lopts="--help --verbose --quiet --distinguished-name --keystore
+ opts="-v -q -d"
+ lopts="--verbose --quiet --distinguished-name --keystore
--repo-keyalias --android-home --no-prompt"
__complete_options
}
local cmd cmds
cmd=${COMP_WORDS[1]}
cmds=" build init install update publish checkupdates import \
-readmeta rewritemeta lint scanner verify stats server "
+readmeta rewritemeta lint scanner verify stats server signindex "
for c in $cmds; do eval "_fdroid_${c} () {
local cur prev opts lopts
@copying
This manual is for the F-Droid repository server tools.
-Copyright @copyright{} 2010, 2011, 2012, 2013 Ciaran Gultnieks
+Copyright @copyright{} 2010, 2011, 2012, 2013, 2014, 2015 Ciaran Gultnieks
Copyright @copyright{} 2011 Henrik Tunedal, Michael Haas, John Sullivan
GNU/Linux
@item
Python 2.x
+To be sure of being able to process all apk files without error, you need
+2.7.7 or later. See @code{http://bugs.python.org/issue14315}.
@item
The Android SDK Tools and Build-tools.
Note that F-Droid does not assume that you have the Android SDK in your
@item
Ruby (debian packages ruby and rubygems)
@item
-Vagrant (unpackaged) Be sure to use 1.3.x because 1.4.x is completely broken
-(at the time of writing, the forthcoming 1.4.3 might work)
+Vagrant (unpackaged, tested on v1.4.3)
@item
vagrant-cachier plugin (unpackaged): `vagrant plugin install vagrant-cachier`
@item
command, without changing the functional content, by running:
@example
-fdroid rewritemetadata
+fdroid rewritemeta
@end example
The following sections describe the fields recognised within the file.
* Web Site::
* Source Code::
* Issue Tracker::
+* Changelog::
* Donate::
* FlattrID::
* Bitcoin::
This is converted to (@code{<tracker>}) in the public index file.
+@node Changelog
+@section Changelog
+
+@cindex Changelog
+
+The URL for the application's changelog. Optional, since not all
+applications have one.
+
+This is converted to (@code{<changelog>}) in the public index file.
+
@node Donate
@section Donate
Here's an example of a complex git-svn Repo URL:
http://svn.code.sf.net/p/project/code/svn;trunk=trunk;tags=tags;branches=branches
-For a Subversion repo that requires authentication, you can precede the repo
-URL with username:password@ and those parameters will be passed as @option{--username}
-and @option{--password} to the SVN checkout command. (This now works for both
-svn and git-svn)
-
If the Repo Type is @code{srclib}, then you must specify the name of the
according srclib .txt file. For example if @code{scrlibs/FooBar.txt} exist
and you want to use this srclib, then you have to set Repo to
takes place.
You can use $$SDK$$, $$NDK$$ and $$MVN3$$ to substitute the paths to the
-android SDK and NDK directories, and maven 3 executable respectively.
+android SDK and NDK directories, and maven 3 executable respectively. The
+following per-build variables are available likewise: $$VERSION$$,
+$$VERCODE$$ and $$COMMIT$$.
@item oldsdkloc=yes
The sdk location in the repo is in an old format, or the build.xml is
If specified, the package version code in the AndroidManifest.xml is
replaced with the version code for the build. See also forceversion.
-@item rm=relpath1,relpath2,...
+@item rm=<path1>[,<path2>,...]
Specifies the relative paths of files or directories to delete before
the build is done. The paths are relative to the base of the build
directory - i.e. the root of the directory structure checked out from
Multiple files/directories can be specified by separating them with ','.
Directories will be recursively deleted.
-@item extlibs=a,b,...
+@item extlibs=<lib1>[,<lib2>,...]
Comma-separated list of external libraries (jar files) from the
@code{build/extlib} library, which will be placed in the @code{libs} directory
of the project.
You can use $$SDK$$, $$NDK$$ and $$MVN3$$ to substitute the paths to the
android SDK and NDK directories, and Maven 3 executable respectively e.g.
-for when you need to run @code{android update project} explicitly.
+for when you need to run @code{android update project} explicitly. The
+following per-build variables are available likewise: $$VERSION$$, $$VERCODE$$
+and $$COMMIT$$.
-@item scanignore=path1,path2,...
+@item scanignore=<path1>[,<path2>,...]
Enables one or more files/paths to be excluded from the scan process.
This should only be used where there is a very good reason, and
probably accompanied by a comment explaining why it is necessary.
When scanning the source tree for problems, matching files whose relative
paths start with any of the paths given here are ignored.
-@item scandelete=path1,path2,...
+@item scandelete=<path1>[,<path2>,...]
Similar to scanignore=, but instead of ignoring files under the given paths,
it tells f-droid to delete the matching files directly.
build= (or the final build) is run.
You can use $$SDK$$, $$NDK$$ and $$MVN3$$ to substitute the paths to the
-android SDK and NDK directories, and Maven 3 executable respectively.
+android SDK and NDK directories, and maven 3 executable respectively. The
+following per-build variables are available likewise: $$VERSION$$,
+$$VERCODE$$ and $$COMMIT$$.
@item buildjni=[yes|no|<dir list>]
Enables building of native code via the ndk-build script before doing
isn't used nor built will result in an error saying that native
libraries were expected in the resulting package.
-@item gradle=<flavour>
-Build with Gradle instead of Ant, specifying what flavour to assemble.
-If <flavour> is 'yes' or 'main', no flavour will be used. Note
-that this will not work on projects with flavours, since it will build
-all flavours and there will be no 'main' build.
+@item ndk=<version>
+Version of the NDK to use in this build. Defaults to the latest NDK release
+that included legacy toolchains, so as to not break builds that require
+toolchains no longer included in current versions of the NDK.
+
+The buildserver supports r9b with its legacy toolchains and the latest release
+as of writing this document, r10e. You may add support for more versions by
+adding them to 'ndk_paths' in your config file.
+
+@item gradle=<flavour1>[,<flavour2>,...]
+Build with Gradle instead of Ant, specifying what flavours to use. Flavours
+are case sensitive since the path to the output apk is as well.
+
+If only one flavour is given and it is 'yes' or 'main', no flavour will be
+used. Note that for projects with flavours, you must specify at least one
+valid flavour since 'yes' or 'main' will build all of them separately.
@item maven=yes[@@<dir>]
Build with Maven instead of Ant. An extra @@<dir> tells f-droid to run Maven
inside that relative subdirectory. Sometimes it is needed to use @@.. so that
builds happen correctly.
-@item preassemble=<task1> <task2>
-Space-separated list of Gradle tasks to be run before the assemble task
-in a Gradle project build.
+@item preassemble=<task1>[,<task2>,...]
+List of Gradle tasks to be run before the assemble task in a Gradle project
+build.
-@item antcommand=xxx
-Specify an alternate Ant command (target) instead of the default
+@item gradleprops=<prop1>[,<prop2>,...]
+List of Gradle properties to pass via the command line to Gradle. A property
+can be of the form @code{foo} or of the form @code{key=value}.
+
+For example: @code{gradleprops=enableFoo,someSetting=bar} will result in
+@code{gradle -PenableFoo -PsomeSetting=bar}.
+
+@item antcommands=<target1>[,<target2>,...]
+Specify an alternate set of Ant commands (target) instead of the default
'release'. It can't be given any flags, such as the path to a build.xml.
@item output=path/to/output.apk
This is optional - if present, it contains a comma-separated list of any of
the following values, describing an anti-feature the application has.
-Even though such apps won't be displayed unless a settings box is ticked,
-it is a good idea to mention the reasons for the anti-feature(s) in the
+It is a good idea to mention the reasons for the anti-feature(s) in the
description:
@itemize @bullet
without major changes to the app.
@item
-@samp{NonFreeAdd} - the application promotes non-Free add-ons, such that the
+@samp{NonFreeAdd} - the application promotes non-free add-ons, such that the
app is effectively an advert for other non-free software and such software is
not clearly labelled as such.
@item
-@samp{NonFreeDep} - the application depends on a non-Free application (e.g.
+@samp{NonFreeDep} - the application depends on a non-free application (e.g.
Google Maps) - i.e. it requires it to be installed on the device, but does not
include it.
+@item
+@samp{UpstreamNonFree} - the application is or depends on non-free software.
+This does not mean that non-free software is included with the app: Most
+likely, it has been patched in some way to remove the non-free code. However,
+functionality may be missing.
+
@end itemize
@node Disabled
name but change it programmatically in some app flavors, by e.g. appending
".open" or ".free" at the end of the package name.
+You can also use @code{Ignore} to ignore package name searching. This should
+only be used in some specific cases, for example if the app's build.gradle
+file does not contain the package name.
+
@node Update Check Data
@section Update Check Data
This field is normally automatically updated - see Update Check Mode.
+If not set or set to @code{0}, clients will recommend the highest version they
+can, as if the @code{Current Version Code} was infinite.
+
This is converted to (@code{<marketvercode>}) in the public index file.
@node No Source Since
@section Setting up a build server
In addition to the basic setup previously described, you will also need
-a Vagrant-compatible Debian Testing base box called 'testing32' (or testing64
+a Vagrant-compatible Debian Testing base box called 'jessie32' (or jessie64
for a 64-bit VM, if you want it to be much slower, and require more disk
space).
for example, having subversion v1.6 on the host and v1.7 on the guest
would fail.
-Unless you're very trusting. you should create one of these for yourself
-from verified standard Debian installation media. However, you could skip
-over the next few paragraphs (and sacrifice some security) by downloading
-@url{https://f-droid.org/testing32.box}.
+@subsection Creating the Debian base box
+
+The output of this step is a minimal Debian VM that has support for remote
+login and provisioning.
+
+Unless you're very trusting, you should create one of these for yourself
+from verified standard Debian installation media. However, by popular
+demand, the @code{makebuildserver} script will automatically download a
+prebuilt image unless instructed otherwise. If you choose to use the
+prebuilt image, you may safely skip the rest of this section.
Documentation for creating a base box can be found at
@url{http://docs.vagrantup.com/v1/docs/base_boxes.html}.
-1 in @code{/etc/grub/default} and then run @code{update-grub}.
@end enumerate
+@subsection Creating the F-Droid base box
-With this base box available, you should then create @code{makebs.config.py},
+The next step in the process is to create @code{makebs.config.py},
using @code{./examples/makebs.config.py} as a reference - look at the settings and
documentation there to decide if any need changing to suit your environment.
There is a path for retrieving the base box if it doesn't exist, and an apt
running the android tools. For example, if you have
@code{buildserver/addons/cache/platforms/android-19.tar.gz} that will be
used when installing the android-19 platform, instead of re-downloading it
-using @code{android update sdk --no-ui -t android-19}.
+using @code{android update sdk --no-ui -t android-19}. It is possible to
+create the cache files of this additions from a local installation of the
+SDK including these:
+
+@example
+cd /path/to/android-sdk/platforms
+tar czf android-19.tar.gz android-19
+mv android-19.tar.gz /path/to/buildserver/addons/cache/platforms/
+@end example
+
+If you have already built a buildserver it is also possible to get this
+files directly from the buildserver:
+
+@example
+vagrant ssh -- -C 'tar -C ~/android-sdk/platforms czf android-19.tar.gz android-19'
+vagrant ssh -- -C 'cat ~/android-sdk/platforms/android-19.tar.gz' > /path/to/fdroidserver/buildserver/cache/platforms/android19.tar.gz
+@end example
Once it's complete you'll have a new base box called 'buildserver' which is
what's used for the actual builds. You can then build packages as normal,
+
+
#!/bin/sh -e
# gendocs.sh -- generate a GNU manual in many formats. This script is
# mentioned in maintain.texi. See the help message below for usage details.
-scriptversion=2013-02-03.15
+scriptversion=2014-10-09.23
# Copyright 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013
# Free Software Foundation, Inc.
cmd="$SETLANG $MAKEINFO -o $PACKAGE.info $commonarg $infoarg \"$srcfile\""
echo "Generating info... ($cmd)"
eval "$cmd"
-tar czf "$outdir/$PACKAGE.info.tar.gz" $PACKAGE.info*
+tar --create $PACKAGE.info* | gzip --no-name -f -9 --to-stdout > "$outdir/$PACKAGE.info.tar.gz"
ls -l "$outdir/$PACKAGE.info.tar.gz"
info_tgz_size=`calcsize "$outdir/$PACKAGE.info.tar.gz"`
# do not mv the info files, there's no point in having them available
printf "\nGenerating dvi... ($cmd)\n"
eval "$cmd"
# compress/finish dvi:
-gzip -f -9 $PACKAGE.dvi
+gzip --no-name -f -9 $PACKAGE.dvi
dvi_gz_size=`calcsize $PACKAGE.dvi.gz`
mv $PACKAGE.dvi.gz "$outdir/"
ls -l "$outdir/$PACKAGE.dvi.gz"
printf "\nGenerating ascii... ($cmd)\n"
eval "$cmd"
ascii_size=`calcsize $PACKAGE.txt`
- gzip -f -9 -c $PACKAGE.txt >"$outdir/$PACKAGE.txt.gz"
+ gzip --no-name -f -9 -c $PACKAGE.txt >"$outdir/$PACKAGE.txt.gz"
ascii_gz_size=`calcsize "$outdir/$PACKAGE.txt.gz"`
mv $PACKAGE.txt "$outdir/"
ls -l "$outdir/$PACKAGE.txt" "$outdir/$PACKAGE.txt.gz"
(
cd ${split_html_dir} || exit 1
ln -sf ${PACKAGE}.html index.html
- tar -czf "$abs_outdir/${PACKAGE}.html_$1.tar.gz" -- *.html
+ tar --create -- *.html | gzip --no-name -f -9 --to-stdout > "$abs_outdir/${PACKAGE}.html_$1.tar.gz"
)
eval html_$1_tgz_size=`calcsize "$outdir/${PACKAGE}.html_$1.tar.gz"`
rm -f "$outdir"/html_$1/*.html
rm -rf $PACKAGE.html # in case a directory is left over
eval "$cmd"
html_mono_size=`calcsize $PACKAGE.html`
- gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz"
+ gzip --no-name -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz"
html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"`
copy_images "$outdir/" $PACKAGE.html
mv $PACKAGE.html "$outdir/"
copy_images $split_html_dir/ $split_html_dir/*.html
(
cd $split_html_dir || exit 1
- tar -czf "$abs_outdir/$PACKAGE.html_$split.tar.gz" -- *
+ tar --create -- * | gzip --no-name -f -9 --to-stdout > "$abs_outdir/$PACKAGE.html_$split.tar.gz"
)
eval \
html_${split}_tgz_size=`calcsize "$outdir/$PACKAGE.html_$split.tar.gz"`
rm -rf $PACKAGE.html # in case a directory is left over
eval "$cmd"
html_mono_size=`calcsize $PACKAGE.html`
- gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz"
+ gzip --no-name -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz"
html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"`
mv $PACKAGE.html "$outdir/"
(
cd "$d"
srcfiles=`ls -d *.texinfo *.texi *.txi *.eps $source_extra 2>/dev/null` || true
- tar czfh "$abs_outdir/$PACKAGE.texi.tar.gz" $srcfiles
+ tar --create --dereference $srcfiles | gzip --no-name -f -9 --to-stdout > "$abs_outdir/$PACKAGE.texi.tar.gz"
ls -l "$abs_outdir/$PACKAGE.texi.tar.gz"
)
texi_tgz_size=`calcsize "$outdir/$PACKAGE.texi.tar.gz"`
printf "\nGenerating docbook XML... ($cmd)\n"
eval "$cmd"
docbook_xml_size=`calcsize $PACKAGE-db.xml`
- gzip -f -9 -c $PACKAGE-db.xml >"$outdir/$PACKAGE-db.xml.gz"
+ gzip --no-name -f -9 -c $PACKAGE-db.xml >"$outdir/$PACKAGE-db.xml.gz"
docbook_xml_gz_size=`calcsize "$outdir/$PACKAGE-db.xml.gz"`
mv $PACKAGE-db.xml "$outdir/"
eval "$cmd"
(
cd ${split_html_db_dir} || exit 1
- tar -czf "$abs_outdir/${PACKAGE}.html_node_db.tar.gz" -- *.html
+ tar --create -- *.html | gzip --no-name -f -9 --to-stdout > "$abs_outdir/${PACKAGE}.html_node_db.tar.gz"
)
html_node_db_tgz_size=`calcsize "$outdir/${PACKAGE}.html_node_db.tar.gz"`
rm -f "$outdir"/html_node_db/*.html
# Copy this file to config.py, then amend the settings below according to
# your system configuration.
-# Override the path to the Android SDK, $ANDROID_HOME by default
-# sdk_path = "/path/to/android-sdk"
+# Custom path to the Android SDK, defaults to $ANDROID_HOME
+# sdk_path = "/opt/android-sdk"
+
+# Custom paths to various versions of the Android NDK, defaults to 'r10e' set
+# to $ANDROID_NDK. Most users will have the latest at $ANDROID_NDK, which is
+# used by default. If a version is missing or assigned to None, it is assumed
+# not installed.
+# ndk_paths = {
+# 'r9b': "/opt/android-ndk-r9b",
+# 'r10e': "/opt/android-ndk",
+# }
-# Override the path to the Android NDK, $ANDROID_NDK by default
-# ndk_path = "/path/to/android-ndk"
# Build tools version to be used
-build_tools = "20.0.0"
+build_tools = "22.0.1"
-# Command for running Ant
-# ant = "/path/to/ant"
+# Command or path to binary for running Ant
ant = "ant"
-# Command for running maven 3
-# mvn3 = "/path/to/mvn"
+# Command or path to binary for running maven 3
mvn3 = "mvn"
-# Command for running Gradle
-# gradle = "/path/to/gradle"
+# Command or path to binary for running Gradle
gradle = "gradle"
# Set the maximum age (in days) of an index that a client should accept from
repo_maxage = 0
repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo"
-repo_name = "My First FDroid Repo Demo"
+repo_name = "My First F-Droid Repo Demo"
repo_icon = "fdroid-icon.png"
repo_description = """
-This is a repository of apps to be used with FDroid. Applications in this
+This is a repository of apps to be used with F-Droid. Applications in this
repository are either official binaries built by the original application
developers, or are binaries built from source by the admin of f-droid.org
using the tools on https://gitlab.com/u/fdroid.
# repository, and no need to define the other archive_ values.
archive_older = 3
archive_url = "https://f-droid.org/archive"
-archive_name = "My First FDroid Archive Demo"
+archive_name = "My First F-Droid Archive Demo"
archive_icon = "fdroid-icon.png"
archive_description = """
The repository of older versions of applications from the main demo repository.
"""
+# `fdroid update` will create a link to the current version of a given app.
+# This provides a static path to the current APK. To disable the creation of
+# this link, uncomment this:
+# make_current_version_link = False
+
+# By default, the "current version" link will be based on the "Name" of the
+# app from the metadata. You can change it to use a different field from the
+# metadata here:
+# current_version_name_source = 'id'
+
+# Optionally, override home directory for gpg
+# gpghome = /home/fdroid/somewhere/else/.gnupg
# The ID of a GPG key for making detached signatures for apks. Optional.
# gpgkey = '1DBA2E89'
# jarsigner using -alias. (Not needed in an unsigned repository).
# repo_keyalias = "fdroidrepo"
+# Optionally, the public key for the key defined by repo_keyalias above can
+# be specified here. There is no need to do this, as the public key can and
+# will be retrieved from the keystore when needed. However, specifying it
+# manually can allow some processing to take place without access to the
+# keystore.
+# repo_pubkey = "..."
+
# The keystore to use for release keys when building. This needs to be
# somewhere safe and secure, and backed up! The best way to manage these
# sensitive keys is to use a "smartcard" (aka Hardware Security Module). To
-# configure FDroid to use a smartcard, set the keystore file using the keyword
+# configure F-Droid to use a smartcard, set the keystore file using the keyword
# "NONE" (i.e. keystore = "NONE"). That makes Java find the keystore on the
# smartcard based on 'smartcardoptions' below.
# keystore = "~/.local/share/fdroidserver/keystore.jks"
# Only the fields listed here are supported, defaults shown
char_limits = {
'Summary': 50,
- 'Description': 1500
+ 'Description': 1500,
}
# You may want to alter these before running ./makebuildserver
# Name of the base box to use
-basebox = "testing32"
+basebox = "jessie32"
-# Location where raring32.box can be found, if you don't already have
+# Location where testing32.box can be found, if you don't already have
# it. For security reasons, it's recommended that you make your own
# in a secure environment using trusted media (see the manual) but
# you can use this default if you like...
-baseboxurl = "https://f-droid.org/testing32.box"
+baseboxurl = "https://f-droid.org/jessie32.box"
+# The amount of RAM the build server will have
memory = 3584
+# The number of CPUs the build server will have
+cpus = 1
+
# Debian package proxy server - if you have one, e.g. "http://192.168.0.19:8000"
aptproxy = None
#!/bin/bash
#
-# fd-commit - part of the FDroid server tools
+# fd-commit - part of the F-Droid server tools
# Commits updates to apps, allowing you to edit the commit messages
#
# Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
disable=false
while read line; do
case "$line" in
- *'Maintainer Notes:'*) break ;;
'-Build:'*) onlybuild=false ;;
'+Build:'*)
$newbuild && onlybuild=false
# -*- coding: utf-8 -*-
#
# fdroid.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
"scanner": "Scan the source code of a package",
"stats": "Update the stats of the repo",
"server": "Interact with the repo HTTP server",
- }
+ "signindex": "Sign indexes created using update --nosign",
+}
def print_help():
import common
import metadata
-from common import FDroidException, BuildException, VCSException, FDroidPopen, SilentPopen
+from common import FDroidException, BuildException, VCSException, FDroidPopen, SdkToolsPopen
try:
import paramiko
if os.path.isdir(vd):
# Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
with open(os.path.join(vd, 'machines', 'default',
- 'virtualbox', 'id')) as vf:
+ 'virtualbox', 'id')) as vf:
id = vf.read()
return id
else:
return True
# Vagrant 1.2 - the directory can exist, but the id can be missing...
if not os.path.exists(os.path.join(vd, 'machines', 'default',
- 'virtualbox', 'id')):
+ 'virtualbox', 'id')):
return False
return True
shutil.rmtree('builder')
os.mkdir('builder')
- p = subprocess.Popen('vagrant --version', shell=True,
+ p = subprocess.Popen(['vagrant', '--version'],
stdout=subprocess.PIPE)
vver = p.communicate()[0]
if vver.startswith('Vagrant version 1.2'):
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, 'metadata.py'), 'metadata.py')
ftp.put(os.path.join(serverpath, '..', 'buildserver',
- 'config.buildserver.py'), 'config.py')
+ 'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
# Copy over the ID (head commit hash) of the fdroidserver in use...
if thisbuild['srclibs']:
for lib in thisbuild['srclibs']:
srclibpaths.append(
- common.getsrclib(lib, 'build/srclib', srclibpaths,
- basepath=True, prepare=False))
+ common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
def adapt_gradle(build_dir):
+ filename = 'build.gradle'
for root, dirs, files in os.walk(build_dir):
- if 'build.gradle' in files:
- path = os.path.join(root, 'build.gradle')
- logging.debug("Adapting build.gradle at %s" % path)
-
- FDroidPopen(['sed', '-i',
- r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"'
- + config['build_tools'] + '"@g', path])
-
-
-def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver):
+ for filename in files:
+ if not filename.endswith('.gradle'):
+ continue
+ path = os.path.join(root, filename)
+ if not os.path.isfile(path):
+ continue
+ logging.debug("Adapting %s at %s" % (filename, path))
+ common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+)['"].*""",
+ r"""\1buildToolsVersion\2'%s'""" % config['build_tools'],
+ path)
+
+
+def capitalize_intact(string):
+ """Like str.capitalize(), but leave the rest of the string intact without
+ switching it to lowercase."""
+ if len(string) == 0:
+ return string
+ if len(string) == 1:
+ return string.upper()
+ return string[0].upper() + string[1:]
+
+
+def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
- if not config['ndk_path']:
- logging.critical("$ANDROID_NDK is not set!")
+ if not thisbuild['ndk_path']:
+ logging.critical("Android NDK version '%s' could not be found!" % thisbuild['ndk'])
+ logging.critical("Configured versions:")
+ for k, v in config['ndk_paths'].iteritems():
+ if k.endswith("_orig"):
+ continue
+ logging.critical(" %s: %s" % (k, v))
sys.exit(3)
- elif not os.path.isdir(config['sdk_path']):
- logging.critical("$ANDROID_NDK points to a non-existing directory!")
+ elif not os.path.isdir(thisbuild['ndk_path']):
+ logging.critical("Android NDK '%s' is not a directory!" % thisbuild['ndk_path'])
sys.exit(3)
+ # Set up environment vars that depend on each build
+ for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
+ common.env[n] = thisbuild['ndk_path']
+
+ common.reset_env_path()
+ # Set up the current NDK to the PATH
+ common.add_to_env_path(thisbuild['ndk_path'])
+
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir,
- extlib_dir, onserver)
+ extlib_dir, onserver, refresh)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
+ gradletasks = []
if thisbuild['type'] == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
elif thisbuild['type'] == 'gradle':
logging.info("Cleaning Gradle project...")
- cmd = [config['gradle'], 'clean']
+
+ if thisbuild['preassemble']:
+ gradletasks += thisbuild['preassemble']
+
+ flavours = thisbuild['gradle']
+ if flavours == ['yes']:
+ flavours = []
+
+ flavours_cmd = ''.join([capitalize_intact(f) for f in flavours])
+
+ gradletasks += ['assemble' + flavours_cmd + 'Release']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
+ cmd = [config['gradle']]
+ if thisbuild['gradleprops']:
+ cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
+
+ for task in gradletasks:
+ parts = task.split(':')
+ parts[-1] = 'clean' + capitalize_intact(parts[-1])
+ cmd += [':'.join(parts)]
+
+ cmd += ['clean']
+
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'kivy':
if 'gradle' in dirs:
shutil.rmtree(os.path.join(root, 'gradle'))
- if not options.skipscan:
+ if options.skipscan:
+ if thisbuild['scandelete']:
+ raise BuildException("Refusing to skip source scan since scandelete is present")
+ else:
# Scan before building...
logging.info("Scanning source for common problems...")
count = common.scan_source(build_dir, root_dir, thisbuild)
if count > 0:
if force:
- logging.warn('Scanner found %d problems:' % count)
+ logging.warn('Scanner found %d problems' % count)
else:
raise BuildException("Can't build due to %d errors while scanning" % count)
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
- if onserver:
- manifest = os.path.join(root_dir, 'AndroidManifest.xml')
- if os.path.exists(manifest):
- homedir = os.path.expanduser('~')
- with open(os.path.join(homedir, 'buildserverid'), 'r') as f:
- buildserverid = f.read()
- with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f:
- fdroidserverid = f.read()
- with open(manifest, 'r') as f:
- manifestcontent = f.read()
- manifestcontent = manifestcontent.replace('</manifest>',
- '<fdroid buildserverid="'
- + buildserverid + '"'
- + ' fdroidserverid="'
- + fdroidserverid + '"'
- + '/></manifest>')
- with open(manifest, 'w') as f:
- f.write(manifestcontent)
-
# Run a build command if one is required...
if thisbuild['build']:
logging.info("Running 'build' commands in %s" % root_dir)
- cmd = common.replace_config_vars(thisbuild['build'])
+ cmd = common.replace_config_vars(thisbuild['build'], thisbuild)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
if jni_components == ['yes']:
jni_components = ['']
- cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"]
+ cmd = [os.path.join(thisbuild['ndk_path'], "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
'package']
if thisbuild['target']:
target = thisbuild["target"].split('-')[1]
- FDroidPopen(['sed', '-i',
- 's@<platform>[0-9]*</platform>@<platform>'
- + target + '</platform>@g',
- 'pom.xml'],
- cwd=root_dir)
+ common.regsub_file(r'<platform>[0-9]*</platform>',
+ r'<platform>%s</platform>' % target,
+ os.path.join(root_dir, 'pom.xml'))
if '@' in thisbuild['maven']:
- FDroidPopen(['sed', '-i',
- 's@<platform>[0-9]*</platform>@<platform>'
- + target + '</platform>@g',
- 'pom.xml'],
- cwd=maven_dir)
+ common.regsub_file(r'<platform>[0-9]*</platform>',
+ r'<platform>%s</platform>' % target,
+ os.path.join(maven_dir, 'pom.xml'))
p = FDroidPopen(mvncmd, cwd=maven_dir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
- cmd += ' ANDROIDNDK=' + config['ndk_path']
- cmd += ' ANDROIDNDKVER=r9'
+ cmd += ' ANDROIDNDK=' + thisbuild['ndk_path']
+ cmd += ' ANDROIDNDKVER=' + thisbuild['ndk']
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
- p = FDroidPopen(cmd, cwd='python-for-android', shell=True)
+ p = subprocess.Popen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
elif thisbuild['type'] == 'gradle':
logging.info("Building Gradle project...")
- flavours = thisbuild['gradle'].split(',')
-
- if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']:
- flavours[0] = ''
-
- commands = [config['gradle']]
- if thisbuild['preassemble']:
- commands += thisbuild['preassemble'].split()
-
- flavours_cmd = ''.join(flavours)
- if flavours_cmd:
- flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:]
-
- commands += ['assemble' + flavours_cmd + 'Release']
# Avoid having to use lintOptions.abortOnError false
if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
f.write("\nandroid { lintOptions { checkReleaseBuilds false } }\n")
- p = FDroidPopen(commands, cwd=root_dir)
+ cmd = [config['gradle']]
+ if thisbuild['gradleprops']:
+ cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
+
+ cmd += gradletasks
+
+ p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
- if thisbuild['antcommand']:
- cmd += [thisbuild['antcommand']]
+ if thisbuild['antcommands']:
+ cmd += thisbuild['antcommands']
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
- p = SilentPopen([config['aapt'], 'dump', 'badging', src])
+ p = SdkToolsPopen(['aapt', 'dump', 'badging', src], output=False)
vercode = None
version = None
str(thisbuild['vercode']))
)
+ # Add information for 'fdroid verify' to be able to reproduce the build
+ # environment.
+ if onserver:
+ metadir = os.path.join(tmp_dir, 'META-INF')
+ if not os.path.exists(metadir):
+ os.mkdir(metadir)
+ homedir = os.path.expanduser('~')
+ for fn in ['buildserverid', 'fdroidserverid']:
+ shutil.copyfile(os.path.join(homedir, fn),
+ os.path.join(metadir, fn))
+ subprocess.call(['jar', 'uf', os.path.abspath(src),
+ 'META-INF/' + fn], cwd=tmp_dir)
+
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
- tmp_dir, repo_dir, vcs, test, server, force, onserver):
+ tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
"""
Build a particular version of an application, if it needs building.
build_server(app, thisbuild, vcs, build_dir, output_dir, force)
else:
- build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver)
+ build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
return True
help="Skip scanning the source code for binaries and other problems")
parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False,
help="Don't create a source tarball, useful when testing a build")
+ parser.add_option("--no-refresh", dest="refresh", action="store_false", default=True,
+ help="Don't refresh the repository, useful when testing a build with no internet connection")
parser.add_option("-f", "--force", action="store_true", default=False,
help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
parser.add_option("-a", "--all", action="store_true", default=False,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
- options.onserver):
+ options.onserver, options.refresh):
+
+ if app.get('Binaries', None):
+ # This is an app where we build from source, and
+ # verify the apk contents against a developer's
+ # binary. We get that binary now, and save it
+ # alongside our built one in the 'unsigend'
+ # directory.
+ url = app['Binaries']
+ url = url.replace('%v', thisbuild['version'])
+ url = url.replace('%c', str(thisbuild['vercode']))
+ logging.info("...retrieving " + url)
+ of = "{0}_{1}.apk.binary".format(app['id'], thisbuild['vercode'])
+ of = os.path.join(output_dir, of)
+ common.download_file(url, local_filename=of)
+
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
# -*- coding: utf-8 -*-
#
# checkupdates.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
return (None, msg)
+def app_matches_packagename(app, package):
+ if not package:
+ return False
+ appid = app['Update Check Name'] or app['id']
+ if appid == "Ignore":
+ return True
+ return appid == package
+
+
# Check for a new version by looking at the tags in the source repo.
# Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
-# Returns (None, "a message") if this didn't work, or (version, vercode) for
+# Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
# the details of the current version.
def check_tags(app, pattern):
try:
- appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
vcs.gotorevision(None)
- flavour = None
+ flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
- flavour = app['builds'][-1]['gradle']
- if flavour == 'yes':
- flavour = None
+ flavours = app['builds'][-1]['gradle']
hpak = None
htag = None
hcode = "0"
tags = vcs.gettags()
+ logging.debug("All tags: " + ','.join(tags))
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
+ logging.debug("Matching tags: " + ','.join(tags))
if repotype in ('git',):
tags = vcs.latesttags(tags, 5)
+ logging.debug("Latest tags: " + ','.join(tags))
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
- paths = common.manifest_paths(build_dir, flavour)
+ paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
- if not package or package != appid or not version or not vercode:
+ if not app_matches_packagename(app, package) or not version or not vercode:
continue
logging.debug("Manifest exists. Found version {0} ({1})"
try:
- appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
elif repotype == 'bzr':
vcs.gotorevision(None)
- flavour = None
-
+ flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
- flavour = app['builds'][-1]['gradle']
- if flavour == 'yes':
- flavour = None
+ flavours = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
- paths = common.manifest_paths(build_dir, flavour)
+ paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package:
return (None, "Couldn't find package ID")
- if package != appid:
- return (None, "Package ID mismatch")
+ if not app_matches_packagename(app, package):
+ return (None, "Package ID mismatch - got {0}".format(package))
if not version:
return (None, "Couldn't find latest version name")
if not vercode:
# subdir relative to the build dir if found, None otherwise.
def check_changed_subdir(app):
- appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
else:
if not os.path.isdir(build_dir):
return None
- flavour = None
+ flavours = []
if len(app['builds']) > 0 and app['builds'][-1]['gradle']:
- flavour = app['builds'][-1]['gradle']
- if flavour == 'yes':
- flavour = None
+ flavours = app['builds'][-1]['gradle']
for d in dirs_with_manifest(build_dir):
logging.debug("Trying possible dir %s." % d)
- m_paths = common.manifest_paths(d, flavour)
+ m_paths = common.manifest_paths(d, flavours)
package = common.parse_androidmanifests(m_paths, app['Update Check Ignore'])[2]
- if package and package == appid:
+ if app_matches_packagename(app, package):
logging.debug("Manifest exists in possible dir %s." % d)
return os.path.relpath(d, build_dir)
except VCSException:
return None
- flavour = None
+ flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
- flavour = app['builds'][-1]['gradle']
- if flavour == 'yes':
- flavour = None
+ flavours = app['builds'][-1]['gradle']
- logging.debug("...fetch auto name from " + app_dir +
- ((" (flavour: %s)" % flavour) if flavour else ""))
- new_name = common.fetch_real_name(app_dir, flavour)
+ logging.debug("...fetch auto name from " + app_dir)
+ new_name = common.fetch_real_name(app_dir, flavours)
commitmsg = None
if new_name:
logging.debug("...got autoname '" + new_name + "'")
else:
logging.debug("...couldn't get autoname")
- if app['Current Version'].startswith('@string/'):
- cv = common.version_name(app['Current Version'], app_dir, flavour)
- if app['Current Version'] != cv:
- app['Current Version'] = cv
- if not commitmsg:
- commitmsg = "Fix CV of {0}".format(common.getappname(app))
-
return commitmsg
import re
import shutil
import glob
+import requests
import stat
import subprocess
import time
import operator
import Queue
import threading
-import magic
import logging
+import hashlib
+import socket
+import xml.etree.ElementTree as XMLElementTree
+
from distutils.version import LooseVersion
+from zipfile import ZipFile
import metadata
+XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
+
config = None
options = None
env = None
+orig_path = None
+
+
+default_config = {
+ 'sdk_path': "$ANDROID_HOME",
+ 'ndk_paths': {
+ 'r9b': None,
+ 'r10e': "$ANDROID_NDK"
+ },
+ 'build_tools': "23.0.0",
+ 'ant': "ant",
+ 'mvn3': "mvn",
+ 'gradle': 'gradle',
+ 'sync_from_local_copy_dir': False,
+ 'make_current_version_link': True,
+ 'current_version_name_source': 'Name',
+ 'update_stats': False,
+ 'stats_ignore': [],
+ 'stats_server': None,
+ 'stats_user': None,
+ 'stats_to_carbon': False,
+ 'repo_maxage': 0,
+ 'build_server_always': False,
+ 'keystore': 'keystore.jks',
+ 'smartcardoptions': [],
+ 'char_limits': {
+ 'Summary': 80,
+ 'Description': 4000
+ },
+ 'keyaliases': {},
+ 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
+ 'repo_name': "My First FDroid Repo Demo",
+ 'repo_icon': "fdroid-icon.png",
+ 'repo_description': '''
+ This is a repository of apps to be used with FDroid. Applications in this
+ repository are either official binaries built by the original application
+ developers, or are binaries built from source by the admin of f-droid.org
+ using the tools on https://gitlab.com/u/fdroid.
+ ''',
+ 'archive_older': 0,
+}
+
+
+def fill_config_defaults(thisconfig):
+ for k, v in default_config.items():
+ if k not in thisconfig:
+ thisconfig[k] = v
+
+ # Expand paths (~users and $vars)
+ def expand_path(path):
+ if path is None:
+ return None
+ orig = path
+ path = os.path.expanduser(path)
+ path = os.path.expandvars(path)
+ if orig == path:
+ return None
+ return path
+
+ for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
+ v = thisconfig[k]
+ exp = expand_path(v)
+ if exp is not None:
+ thisconfig[k] = exp
+ thisconfig[k + '_orig'] = v
+ for k in ['ndk_paths']:
+ d = thisconfig[k]
+ for k2 in d.copy():
+ v = d[k2]
+ exp = expand_path(v)
+ if exp is not None:
+ thisconfig[k][k2] = exp
+ thisconfig[k][k2 + '_orig'] = v
-def get_default_config():
- return {
- 'sdk_path': os.getenv("ANDROID_HOME") or "",
- 'ndk_path': os.getenv("ANDROID_NDK") or "",
- 'build_tools': "20.0.0",
- 'ant': "ant",
- 'mvn3': "mvn",
- 'gradle': 'gradle',
- 'sync_from_local_copy_dir': False,
- 'update_stats': False,
- 'stats_ignore': [],
- 'stats_server': None,
- 'stats_user': None,
- 'stats_to_carbon': False,
- 'repo_maxage': 0,
- 'build_server_always': False,
- 'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
- 'smartcardoptions': [],
- 'char_limits': {
- 'Summary': 50,
- 'Description': 1500
- },
- 'keyaliases': {},
- 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
- 'repo_name': "My First FDroid Repo Demo",
- 'repo_icon': "fdroid-icon.png",
- 'repo_description': '''
- This is a repository of apps to be used with FDroid. Applications in this
- repository are either official binaries built by the original application
- developers, or are binaries built from source by the admin of f-droid.org
- using the tools on https://gitlab.com/u/fdroid.
- ''',
- 'archive_older': 0,
- }
+
+def regsub_file(pattern, repl, path):
+ with open(path, 'r') as f:
+ text = f.read()
+ text = re.sub(pattern, repl, text)
+ with open(path, 'w') as f:
+ f.write(text)
def read_config(opts, config_file='config.py'):
The config is read from config_file, which is in the current directory when
any of the repo management commands are used.
"""
- global config, options, env
+ global config, options, env, orig_path
if config is not None:
return config
if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
- defconfig = get_default_config()
- for k, v in defconfig.items():
- if k not in config:
- config[k] = v
-
- # Expand environment variables
- for k, v in config.items():
- if type(v) != str:
- continue
- v = os.path.expanduser(v)
- config[k] = os.path.expandvars(v)
-
- if not test_sdk_exists(config):
- sys.exit(3)
-
- if not test_build_tools_exists(config):
- sys.exit(3)
-
- bin_paths = {
- 'aapt': [
- os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
- ],
- 'zipalign': [
- os.path.join(config['sdk_path'], 'tools', 'zipalign'),
- os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
- ],
- 'android': [
- os.path.join(config['sdk_path'], 'tools', 'android'),
- ],
- 'adb': [
- os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
- ],
- }
-
- for b, paths in bin_paths.items():
- config[b] = None
- for path in paths:
- if os.path.isfile(path):
- config[b] = path
- break
- if config[b] is None:
- logging.warn("Could not find %s in any of the following paths:\n%s" % (
- b, '\n'.join(paths)))
+ fill_config_defaults(config)
# There is no standard, so just set up the most common environment
# variables
env = os.environ
+ orig_path = env['PATH']
for n in ['ANDROID_HOME', 'ANDROID_SDK']:
env[n] = config['sdk_path']
- for n in ['ANDROID_NDK', 'NDK']:
- env[n] = config['ndk_path']
for k in ["keystorepass", "keypass"]:
if k in config:
return config
-def test_sdk_exists(c):
- if c['sdk_path'] is None:
- # c['sdk_path'] is set to the value of ANDROID_HOME by default
- logging.error('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
+def get_ndk_path(version):
+ if version is None:
+ version = 'r10e' # falls back to latest
+ paths = config['ndk_paths']
+ if version not in paths:
+ return ''
+ return paths[version] or ''
+
+
+def find_sdk_tools_cmd(cmd):
+ '''find a working path to a tool from the Android SDK'''
+
+ tooldirs = []
+ if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
+ # try to find a working path to this command, in all the recent possible paths
+ if 'build_tools' in config:
+ build_tools = os.path.join(config['sdk_path'], 'build-tools')
+ # if 'build_tools' was manually set and exists, check only that one
+ configed_build_tools = os.path.join(build_tools, config['build_tools'])
+ if os.path.exists(configed_build_tools):
+ tooldirs.append(configed_build_tools)
+ else:
+ # no configed version, so hunt known paths for it
+ for f in sorted(os.listdir(build_tools), reverse=True):
+ if os.path.isdir(os.path.join(build_tools, f)):
+ tooldirs.append(os.path.join(build_tools, f))
+ tooldirs.append(build_tools)
+ sdk_tools = os.path.join(config['sdk_path'], 'tools')
+ if os.path.exists(sdk_tools):
+ tooldirs.append(sdk_tools)
+ sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
+ if os.path.exists(sdk_platform_tools):
+ tooldirs.append(sdk_platform_tools)
+ tooldirs.append('/usr/bin')
+ for d in tooldirs:
+ if os.path.isfile(os.path.join(d, cmd)):
+ return os.path.join(d, cmd)
+ # did not find the command, exit with error message
+ ensure_build_tools_exists(config)
+
+
+def test_sdk_exists(thisconfig):
+ if 'sdk_path' not in thisconfig:
+ if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
+ return True
+ else:
+ logging.error("'sdk_path' not set in config.py!")
+ return False
+ if thisconfig['sdk_path'] == default_config['sdk_path']:
+ logging.error('No Android SDK found!')
logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
logging.error('\texport ANDROID_HOME=/opt/android-sdk')
return False
- if not os.path.exists(c['sdk_path']):
- logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
+ if not os.path.exists(thisconfig['sdk_path']):
+ logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
return False
- if not os.path.isdir(c['sdk_path']):
- logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
+ if not os.path.isdir(thisconfig['sdk_path']):
+ logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
return False
for d in ['build-tools', 'platform-tools', 'tools']:
- if not os.path.isdir(os.path.join(c['sdk_path'], d)):
+ if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
- c['sdk_path'], d))
+ thisconfig['sdk_path'], d))
return False
return True
-def test_build_tools_exists(c):
- if not test_sdk_exists(c):
- return False
- build_tools = os.path.join(c['sdk_path'], 'build-tools')
- versioned_build_tools = os.path.join(build_tools, c['build_tools'])
+def ensure_build_tools_exists(thisconfig):
+ if not test_sdk_exists(thisconfig):
+ sys.exit(3)
+ build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
+ versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
if not os.path.isdir(versioned_build_tools):
logging.critical('Android Build Tools path "'
+ versioned_build_tools + '" does not exist!')
- return False
- return True
+ sys.exit(3)
def write_password_file(pwtype, password=None):
class vcs:
+
def __init__(self, remote, local):
# svn, git-svn and bzr may require auth
self.username = None
if self.repotype() in ('git-svn', 'bzr'):
if '@' in remote:
+ if self.repotype == 'git-svn':
+ raise VCSException("Authentication is not supported for git-svn")
self.username, remote = remote.split('@')
if ':' not in self.username:
raise VCSException("Password required with username")
# lifetime of the vcs object.
# None is acceptable for 'rev' if you know you are cloning a clean copy of
# the repo - otherwise it must specify a valid revision.
- def gotorevision(self, rev):
+ def gotorevision(self, rev, refresh=True):
if self.clone_failed:
raise VCSException("Downloading the repository already failed once, not trying again.")
writeback = False
else:
deleterepo = True
- logging.info(
- "Repository details for %s changed - deleting" % (
- self.local))
+ logging.info("Repository details for %s changed - deleting" % (
+ self.local))
else:
deleterepo = True
logging.info("Repository details for %s missing - deleting" % (
shutil.rmtree(self.local)
exc = None
+ if not refresh:
+ self.refreshed = True
try:
self.gotorevisionx(rev)
# Get a list of all known tags
def gettags(self):
- raise VCSException('gettags not supported for this vcs type')
-
- # Get a list of latest number tags
- def latesttags(self, number):
+ if not self._gettags:
+ raise VCSException('gettags not supported for this vcs type')
+ rtags = []
+ for tag in self._gettags():
+ if re.match('[-A-Za-z0-9_. ]+$', tag):
+ rtags.append(tag)
+ return rtags
+
+ def latesttags(self, tags, number):
+ """Get the most recent tags in a given list.
+
+ :param tags: a list of tags
+ :param number: the number to return
+ :returns: A list containing the most recent tags in the provided
+ list, up to the maximum number given.
+ """
raise VCSException('latesttags not supported for this vcs type')
# Get current commit reference (hash, revision, etc)
# fdroidserver) and then we'll proceed to destroy it! This is called as
# a safety check.
def checkrepo(self):
- p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
+ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip()
if not result.endswith(self.local):
raise VCSException('Repository mismatch')
else:
self.checkrepo()
# Discard any working tree changes
- p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git fetch failed", p.output)
- p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
+ p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git fetch failed", p.output)
# Recreate origin/HEAD as git clone would do it, in case it disappeared
- p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
+ p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
if p.returncode != 0:
lines = p.output.splitlines()
if 'Multiple remote HEAD branches' not in lines[0]:
raise VCSException("Git remote set-head failed", p.output)
branch = lines[1].split(' ')[-1]
- p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
+ p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
if p2.returncode != 0:
raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
self.refreshed = True
# origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
# a github repo. Most of the time this is the same as origin/master.
rev = rev or 'origin/HEAD'
- p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git checkout of '%s' failed" % rev, p.output)
# Get rid of any uncontrolled files left behind
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
line = line.replace('git@github.com:', 'https://github.com/')
f.write(line)
- for cmd in [
- ['git', 'reset', '--hard'],
- ['git', 'clean', '-dffx'],
- ]:
- p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
- if p.returncode != 0:
- raise VCSException("Git submodule reset failed", p.output)
- p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git submodule sync failed", p.output)
p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git submodule update failed", p.output)
- def gettags(self):
+ def _gettags(self):
self.checkrepo()
- p = SilentPopen(['git', 'tag'], cwd=self.local)
+ p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
return p.output.splitlines()
- def latesttags(self, alltags, number):
+ def latesttags(self, tags, number):
self.checkrepo()
- p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
- + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
- + 'sort -n | awk \'{print $2}\''],
- cwd=self.local, shell=True)
- return p.output.splitlines()[-number:]
+ tl = []
+ for tag in tags:
+ p = FDroidPopen(
+ ['git', 'show', '--format=format:%ct', '-s', tag],
+ cwd=self.local, output=False)
+ # Timestamp is on the last line. For a normal tag, it's the only
+ # line, but for annotated tags, the rest of the info precedes it.
+ ts = int(p.output.splitlines()[-1])
+ tl.append((ts, tag))
+ latest = []
+ for _, t in sorted(tl)[-number:]:
+ latest.append(t)
+ return latest
class vcs_gitsvn(vcs):
def repotype(self):
return 'git-svn'
- # Damn git-svn tries to use a graphical password prompt, so we have to
- # trick it into taking the password from stdin
- def userargs(self):
- if self.username is None:
- return ('', '')
- return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
-
# If the local directory exists, but is somehow not a git repository, git
# will traverse up the directory tree until it finds one that is (i.e.
# fdroidserver) and then we'll proceed to destory it! This is called as
# a safety check.
def checkrepo(self):
- p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
+ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip()
if not result.endswith(self.local):
raise VCSException('Repository mismatch')
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
# Brand new checkout
- gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
+ gitsvn_args = ['git', 'svn', 'clone']
if ';' in self.remote:
remote_split = self.remote.split(';')
for i in remote_split[1:]:
if i.startswith('trunk='):
- gitsvn_cmd += ' -T %s' % i[6:]
+ gitsvn_args.extend(['-T', i[6:]])
elif i.startswith('tags='):
- gitsvn_cmd += ' -t %s' % i[5:]
+ gitsvn_args.extend(['-t', i[5:]])
elif i.startswith('branches='):
- gitsvn_cmd += ' -b %s' % i[9:]
- p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
+ gitsvn_args.extend(['-b', i[9:]])
+ gitsvn_args.extend([remote_split[0], self.local])
+ p = FDroidPopen(gitsvn_args, output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git svn clone failed", p.output)
else:
- p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
+ gitsvn_args.extend([self.remote, self.local])
+ p = FDroidPopen(gitsvn_args, output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git svn clone failed", p.output)
else:
self.checkrepo()
# Discard any working tree changes
- p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
+ p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
# Get new commits, branches and tags from repo
- p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
+ p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git svn fetch failed")
- p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
+ p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git svn rebase failed", p.output)
self.refreshed = True
nospaces_rev = rev.replace(' ', '%20')
# Try finding a svn tag
for treeish in ['origin/', '']:
- p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
- cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
if p.returncode == 0:
break
if p.returncode != 0:
svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
- p = SilentPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish],
- cwd=self.local)
+ p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
git_rev = p.output.rstrip()
if p.returncode == 0 and git_rev:
if p.returncode != 0 or not git_rev:
# Try a plain git checkout as a last resort
- p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
else:
# Check out the git rev equivalent to the svn rev
- p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git checkout of '%s' failed" % rev, p.output)
# Get rid of any uncontrolled files left behind
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
- def gettags(self):
+ def _gettags(self):
self.checkrepo()
for treeish in ['origin/', '']:
d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
def getref(self):
self.checkrepo()
- p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
+ p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
if p.returncode != 0:
return None
return p.output.strip()
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
- p = SilentPopen(['hg', 'clone', self.remote, self.local])
+ p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Hg clone failed", p.output)
else:
- p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
+ p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
if p.returncode != 0:
- raise VCSException("Hg clean failed", p.output)
+ raise VCSException("Hg status failed", p.output)
+ for line in p.output.splitlines():
+ if not line.startswith('? '):
+ raise VCSException("Unexpected output from hg status -uS: " + line)
+ FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
if not self.refreshed:
- p = SilentPopen(['hg', 'pull'], cwd=self.local)
+ p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Hg pull failed", p.output)
self.refreshed = True
rev = rev or 'default'
if not rev:
return
- p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
+ p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
- p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
+ p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
# Also delete untracked files, we have to enable purge extension for that:
if "'purge' is provided by the following extension" in p.output:
with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
myfile.write("\n[extensions]\nhgext.purge=\n")
- p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
+ p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("HG purge failed", p.output)
elif p.returncode != 0:
raise VCSException("HG purge failed", p.output)
- def gettags(self):
- p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
+ def _gettags(self):
+ p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
return p.output.splitlines()[1:]
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
- p = SilentPopen(['bzr', 'branch', self.remote, self.local])
+ p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Bzr branch failed", p.output)
else:
- p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
+ p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Bzr revert failed", p.output)
if not self.refreshed:
- p = SilentPopen(['bzr', 'pull'], cwd=self.local)
+ p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Bzr update failed", p.output)
self.refreshed = True
revargs = list(['-r', rev] if rev else [])
- p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
+ p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
- def gettags(self):
- p = SilentPopen(['bzr', 'tags'], cwd=self.local)
+ def _gettags(self):
+ p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
return [tag.split(' ')[0].strip() for tag in
p.output.splitlines()]
-def retrieve_string(app_dir, string, xmlfiles=None):
+def unescape_string(string):
+ if string[0] == '"' and string[-1] == '"':
+ return string[1:-1]
+
+ return string.replace("\\'", "'")
+
- res_dirs = [
- os.path.join(app_dir, 'res'),
- os.path.join(app_dir, 'src', 'main'),
- ]
+def retrieve_string(app_dir, string, xmlfiles=None):
if xmlfiles is None:
xmlfiles = []
- for res_dir in res_dirs:
+ for res_dir in [
+ os.path.join(app_dir, 'res'),
+ os.path.join(app_dir, 'src', 'main', 'res'),
+ ]:
for r, d, f in os.walk(res_dir):
if os.path.basename(r) == 'values':
xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
- string_search = None
- if string.startswith('@string/'):
- string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
- elif string.startswith('&') and string.endswith(';'):
- string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
-
- if string_search is not None:
- for xmlfile in xmlfiles:
- for line in file(xmlfile):
- matches = string_search(line)
- if matches:
- return retrieve_string(app_dir, matches.group(1), xmlfiles)
- return None
+ if not string.startswith('@string/'):
+ return unescape_string(string)
- return string.replace("\\'", "'")
+ name = string[len('@string/'):]
+
+ for path in xmlfiles:
+ if not os.path.isfile(path):
+ continue
+ xml = parse_xml(path)
+ element = xml.find('string[@name="' + name + '"]')
+ if element is not None and element.text is not None:
+ return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
+
+ return ''
+
+
+def retrieve_string_singleline(app_dir, string, xmlfiles=None):
+ return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
# Return list of existing files that will be used to find the highest vercode
-def manifest_paths(app_dir, flavour):
+def manifest_paths(app_dir, flavours):
possible_manifests = \
[os.path.join(app_dir, 'AndroidManifest.xml'),
os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
os.path.join(app_dir, 'build.gradle')]
- if flavour:
+ for flavour in flavours:
+ if flavour == 'yes':
+ continue
possible_manifests.append(
os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
# Retrieve the package name. Returns the name, or None if not found.
-def fetch_real_name(app_dir, flavour):
- app_search = re.compile(r'.*<application.*').search
- name_search = re.compile(r'.*android:label="([^"]+)".*').search
- app_found = False
- for f in manifest_paths(app_dir, flavour):
- if not has_extension(f, 'xml'):
+def fetch_real_name(app_dir, flavours):
+ for path in manifest_paths(app_dir, flavours):
+ if not has_extension(path, 'xml') or not os.path.isfile(path):
continue
- logging.debug("fetch_real_name: Checking manifest at " + f)
- for line in file(f):
- if not app_found:
- if app_search(line):
- app_found = True
- if app_found:
- matches = name_search(line)
- if matches:
- stringname = matches.group(1)
- logging.debug("fetch_real_name: using string " + stringname)
- result = retrieve_string(app_dir, stringname)
- if result:
- result = result.strip()
- return result
- return None
-
-
-# Retrieve the version name
-def version_name(original, app_dir, flavour):
- for f in manifest_paths(app_dir, flavour):
- if not has_extension(f, 'xml'):
+ logging.debug("fetch_real_name: Checking manifest at " + path)
+ xml = parse_xml(path)
+ app = xml.find('application')
+ if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
continue
- string = retrieve_string(app_dir, original)
- if string:
- return string
- return original
+ label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+ result = retrieve_string_singleline(app_dir, label)
+ if result:
+ result = result.strip()
+ return result
+ return None
def get_library_references(root_dir):
proppath = os.path.join(root_dir, 'project.properties')
if not os.path.isfile(proppath):
return libraries
- with open(proppath) as f:
- for line in f.readlines():
- if not line.startswith('android.library.reference.'):
- continue
- path = line.split('=')[1].strip()
- relpath = os.path.join(root_dir, path)
- if not os.path.isdir(relpath):
- continue
- logging.debug("Found subproject at %s" % path)
- libraries.append(path)
+ for line in file(proppath):
+ if not line.startswith('android.library.reference.'):
+ continue
+ path = line.split('=')[1].strip()
+ relpath = os.path.join(root_dir, path)
+ if not os.path.isdir(relpath):
+ continue
+ logging.debug("Found subproject at %s" % path)
+ libraries.append(path)
return libraries
logging.debug("Removing debuggable flags from %s" % root_dir)
for root, dirs, files in os.walk(root_dir):
if 'AndroidManifest.xml' in files:
- path = os.path.join(root, 'AndroidManifest.xml')
- p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
- if p.returncode != 0:
- raise BuildException("Failed to remove debuggable flags of %s" % path)
+ regsub_file(r'android:debuggable="[^"]*"',
+ '',
+ os.path.join(root, 'AndroidManifest.xml'))
# Extract some information from the AndroidManifest.xml at the given path.
if not paths:
return (None, None, None)
- vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
- vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
- psearch = re.compile(r'.*package="([^"]+)".*').search
-
vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
for path in paths:
+ if not os.path.isfile(path):
+ continue
+
+ logging.debug("Parsing manifest at {0}".format(path))
gradle = has_extension(path, 'gradle')
version = None
vercode = None
# Remember package name, may be defined separately from version+vercode
package = max_package
- for line in file(path):
- if not package:
- if gradle:
+ if gradle:
+ for line in file(path):
+ if not package:
matches = psearch_g(line)
- else:
- matches = psearch(line)
- if matches:
- package = matches.group(1)
- if not version:
- if gradle:
+ if matches:
+ package = matches.group(1)
+ if not version:
matches = vnsearch_g(line)
- else:
- matches = vnsearch(line)
- if matches:
- version = matches.group(2 if gradle else 1)
- if not vercode:
- if gradle:
+ if matches:
+ version = matches.group(2)
+ if not vercode:
matches = vcsearch_g(line)
- else:
- matches = vcsearch(line)
- if matches:
- vercode = matches.group(1)
+ if matches:
+ vercode = matches.group(1)
+ else:
+ xml = parse_xml(path)
+ if "package" in xml.attrib:
+ package = xml.attrib["package"].encode('utf-8')
+ if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+ version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+ base_dir = os.path.dirname(path)
+ version = retrieve_string_singleline(base_dir, version)
+ if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+ a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+ if string_is_integer(a):
+ vercode = a
+
+ logging.debug("..got package={0}, version={1}, vercode={2}"
+ .format(package, version, vercode))
# Always grab the package name and version name in case they are not
# together with the highest version code
if max_version is None:
max_version = "Unknown"
+ if max_package and not is_valid_package_name(max_package):
+ raise FDroidException("Invalid package name {0}".format(max_package))
+
return (max_version, max_vercode, max_package)
+def is_valid_package_name(name):
+ return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
+
+
class FDroidException(Exception):
+
def __init__(self, value, detail=None):
self.value = value
self.detail = detail
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
-def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
- basepath=False, raw=False, prepare=True, preponly=False):
+def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
+ raw=False, prepare=True, preponly=False, refresh=True):
number = None
subdir = None
vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
vcs.srclib = (name, number, sdir)
if ref:
- vcs.gotorevision(ref)
+ vcs.gotorevision(ref, refresh)
if raw:
return vcs
if libdir is None:
libdir = sdir
- if srclib["Srclibs"]:
- n = 1
- for lib in srclib["Srclibs"].replace(';', ',').split(','):
- s_tuple = None
- for t in srclibpaths:
- if t[0] == lib:
- s_tuple = t
- break
- if s_tuple is None:
- raise VCSException('Missing recursive srclib %s for %s' % (
- lib, name))
- place_srclib(libdir, n, s_tuple[2])
- n += 1
-
remove_signing_keys(sdir)
remove_debuggable_flags(sdir)
if prepare:
if srclib["Prepare"]:
- cmd = replace_config_vars(srclib["Prepare"])
+ cmd = replace_config_vars(srclib["Prepare"], None)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
if p.returncode != 0:
# 'root' is the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
# 'srclibpaths' is information on the srclibs being used
-def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
+def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
# Optionally, the actual app source can be in a subdirectory
if build['subdir']:
# Get a working copy of the right revision
logging.info("Getting source for revision " + build['commit'])
- vcs.gotorevision(build['commit'])
+ vcs.gotorevision(build['commit'], refresh)
- # Initialise submodules if requred
+ # Initialise submodules if required
if build['submodules']:
logging.info("Initialising submodules")
vcs.initsubmodules()
# Run an init command if one is required
if build['init']:
- cmd = replace_config_vars(build['init'])
+ cmd = replace_config_vars(build['init'], build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if build['srclibs']:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
- srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
- preponly=onserver))
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
if build['subdir']:
localprops += [os.path.join(root_dir, 'local.properties')]
for path in localprops:
- if not os.path.isfile(path):
- continue
- logging.info("Updating properties file at %s" % path)
- f = open(path, 'r')
- props = f.read()
- f.close()
- props += '\n'
+ props = ""
+ if os.path.isfile(path):
+ logging.info("Updating local.properties file at %s" % path)
+ f = open(path, 'r')
+ props += f.read()
+ f.close()
+ props += '\n'
+ else:
+ logging.info("Creating local.properties file at %s" % path)
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary
if build['oldsdkloc']:
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
- if 'ndk_path' in config:
+ if build['ndk_path']:
# Add ndk location
- props += "ndk.dir=%s\n" % config['ndk_path']
- props += "ndk-location=%s\n" % config['ndk_path']
+ props += "ndk.dir=%s\n" % build['ndk_path']
+ props += "ndk-location=%s\n" % build['ndk_path']
# Add java.encoding if necessary
if build['encoding']:
props += "java.encoding=%s\n" % build['encoding']
f.write(props)
f.close()
- flavour = None
+ flavours = []
if build['type'] == 'gradle':
- flavour = build['gradle']
- if flavour in ['main', 'yes', '']:
- flavour = None
+ flavours = build['gradle']
- version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
+ version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
gradlepluginver = None
- gradle_files = [os.path.join(root_dir, 'build.gradle')]
+ gradle_dirs = [root_dir]
# Parent dir build.gradle
parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
if parent_dir.startswith(build_dir):
- gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
+ gradle_dirs.append(parent_dir)
- for path in gradle_files:
+ for dir_path in gradle_dirs:
if gradlepluginver:
break
- if not os.path.isfile(path):
+ if not os.path.isdir(dir_path):
continue
- with open(path) as f:
- for line in f:
+ for filename in os.listdir(dir_path):
+ if not filename.endswith('.gradle'):
+ continue
+ path = os.path.join(dir_path, filename)
+ if not os.path.isfile(path):
+ continue
+ for line in file(path):
match = version_regex.match(line)
if match:
gradlepluginver = match.group(1)
if build['target']:
n = build["target"].split('-')[1]
- SilentPopen(['sed', '-i',
- 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
- 'build.gradle'],
- cwd=root_dir)
+ regsub_file(r'compileSdkVersion[ =]+[0-9]+',
+ r'compileSdkVersion %s' % n,
+ os.path.join(root_dir, 'build.gradle'))
# Remove forced debuggable flags
remove_debuggable_flags(root_dir)
# Insert version code and number into the manifest if necessary
if build['forceversion']:
logging.info("Changing the version name")
- for path in manifest_paths(root_dir, flavour):
+ for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = SilentPopen(['sed', '-i',
- 's/android:versionName="[^"]*"/android:versionName="'
- + build['version'] + '"/g',
- path])
- if p.returncode != 0:
- raise BuildException("Failed to amend manifest")
+ regsub_file(r'android:versionName="[^"]*"',
+ r'android:versionName="%s"' % build['version'],
+ path)
elif has_extension(path, 'gradle'):
- p = SilentPopen(['sed', '-i',
- 's/versionName *=* *"[^"]*"/versionName = "'
- + build['version'] + '"/g',
- path])
- if p.returncode != 0:
- raise BuildException("Failed to amend build.gradle")
+ regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
+ r"""\1versionName '%s'""" % build['version'],
+ path)
+
if build['forcevercode']:
logging.info("Changing the version code")
- for path in manifest_paths(root_dir, flavour):
+ for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = SilentPopen(['sed', '-i',
- 's/android:versionCode="[^"]*"/android:versionCode="'
- + build['vercode'] + '"/g',
- path])
- if p.returncode != 0:
- raise BuildException("Failed to amend manifest")
+ regsub_file(r'android:versionCode="[^"]*"',
+ r'android:versionCode="%s"' % build['vercode'],
+ path)
elif has_extension(path, 'gradle'):
- p = SilentPopen(['sed', '-i',
- 's/versionCode *=* *[0-9]*/versionCode = '
- + build['vercode'] + '/g',
- path])
- if p.returncode != 0:
- raise BuildException("Failed to amend build.gradle")
+ regsub_file(r'versionCode[ =]+[0-9]+',
+ r'versionCode %s' % build['vercode'],
+ path)
# Delete unwanted files
if build['rm']:
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
if os.path.islink(dest):
- SilentPopen(['unlink ' + dest], shell=True)
+ FDroidPopen(['unlink', dest], output=False)
else:
- SilentPopen(['rm -rf ' + dest], shell=True)
+ FDroidPopen(['rm', '-rf', dest], output=False)
else:
logging.info("...but it didn't exist")
if build['prebuild']:
logging.info("Running 'prebuild' commands in %s" % root_dir)
- cmd = replace_config_vars(build['prebuild'])
+ cmd = replace_config_vars(build['prebuild'], build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
# Generate (or update) the ant build file, build.xml...
if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
- parms = [config['android'], 'update', 'lib-project']
- lparms = [config['android'], 'update', 'project']
+ parms = ['android', 'update', 'lib-project']
+ lparms = ['android', 'update', 'project']
if build['target']:
parms += ['-t', build['target']]
else:
logging.debug("Updating subproject %s" % d)
cmd = lparms + ['-p', d]
- p = FDroidPopen(cmd, cwd=root_dir)
+ p = SdkToolsPopen(cmd, cwd=root_dir)
# Check to see whether an error was returned without a proper exit
# code (this is the case for the 'no target set or target invalid'
# error)
return paths
+def init_mime_type():
+ '''
+ There are two incompatible versions of the 'magic' module, one
+ that comes as part of libmagic, which is what Debian includes as
+ python-magic, then another called python-magic that is a separate
+ project that wraps libmagic. The second is 'magic' on pypi, so
+ both need to be supported. Then on platforms where libmagic is
+ not easily included, e.g. OSX and Windows, fallback to the
+ built-in 'mimetypes' module so this will work without
+ libmagic. Hence this function with the following hacks:
+ '''
+
+ init_path = ''
+ method = ''
+ ms = None
+
+ def mime_from_file(path):
+ try:
+ return magic.from_file(path, mime=True)
+ except UnicodeError:
+ return None
+
+ def mime_file(path):
+ try:
+ return ms.file(path)
+ except UnicodeError:
+ return None
+
+ def mime_guess_type(path):
+ return mimetypes.guess_type(path, strict=False)
+
+ try:
+ import magic
+ try:
+ ms = magic.open(magic.MIME_TYPE)
+ ms.load()
+ magic.from_file(init_path, mime=True)
+ method = 'from_file'
+ except AttributeError:
+ ms.file(init_path)
+ method = 'file'
+ except ImportError:
+ import mimetypes
+ mimetypes.init()
+ method = 'guess_type'
+
+ logging.info("Using magic method " + method)
+ if method == 'from_file':
+ return mime_from_file
+ if method == 'file':
+ return mime_file
+ if method == 'guess_type':
+ return mime_guess_type
+
+ logging.critical("unknown magic method!")
+
+
# Scan the source code in the given directory (and all subdirectories)
# and return the number of fatal problems encountered
def scan_source(build_dir, root_dir, thisbuild):
# Common known non-free blobs (always lower case):
usual_suspects = [
- re.compile(r'flurryagent', re.IGNORECASE),
- re.compile(r'paypal.*mpl', re.IGNORECASE),
- re.compile(r'google.*analytics', re.IGNORECASE),
- re.compile(r'admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'google.*ad.*view', re.IGNORECASE),
- re.compile(r'google.*admob', re.IGNORECASE),
- re.compile(r'google.*play.*services', re.IGNORECASE),
- re.compile(r'crittercism', re.IGNORECASE),
- re.compile(r'heyzap', re.IGNORECASE),
- re.compile(r'jpct.*ae', re.IGNORECASE),
- re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'bugsense', re.IGNORECASE),
- re.compile(r'crashlytics', re.IGNORECASE),
- re.compile(r'ouya.*sdk', re.IGNORECASE),
- re.compile(r'libspen23', re.IGNORECASE),
- ]
+ re.compile(r'.*flurryagent', re.IGNORECASE),
+ re.compile(r'.*paypal.*mpl', re.IGNORECASE),
+ re.compile(r'.*google.*analytics', re.IGNORECASE),
+ re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
+ re.compile(r'.*google.*ad.*view', re.IGNORECASE),
+ re.compile(r'.*google.*admob', re.IGNORECASE),
+ re.compile(r'.*google.*play.*services', re.IGNORECASE),
+ re.compile(r'.*crittercism', re.IGNORECASE),
+ re.compile(r'.*heyzap', re.IGNORECASE),
+ re.compile(r'.*jpct.*ae', re.IGNORECASE),
+ re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
+ re.compile(r'.*bugsense', re.IGNORECASE),
+ re.compile(r'.*crashlytics', re.IGNORECASE),
+ re.compile(r'.*ouya.*sdk', re.IGNORECASE),
+ re.compile(r'.*libspen23', re.IGNORECASE),
+ ]
scanignore = getpaths(build_dir, thisbuild, 'scanignore')
scandelete = getpaths(build_dir, thisbuild, 'scandelete')
- try:
- ms = magic.open(magic.MIME_TYPE)
- ms.load()
- except AttributeError:
- ms = None
+ scanignore_worked = set()
+ scandelete_worked = set()
def toignore(fd):
- for i in scanignore:
- if fd.startswith(i):
+ for p in scanignore:
+ if fd.startswith(p):
+ scanignore_worked.add(p)
return True
return False
def todelete(fd):
- for i in scandelete:
- if fd.startswith(i):
+ for p in scandelete:
+ if fd.startswith(p):
+ scandelete_worked.add(p)
return True
return False
+ def ignoreproblem(what, fd, fp):
+ logging.info('Ignoring %s at %s' % (what, fd))
+ return 0
+
def removeproblem(what, fd, fp):
logging.info('Removing %s at %s' % (what, fd))
os.remove(fp)
+ return 0
def warnproblem(what, fd):
logging.warn('Found %s at %s' % (what, fd))
def handleproblem(what, fd, fp):
if toignore(fd):
- logging.info('Ignoring %s at %s' % (what, fd))
- elif todelete(fd):
- removeproblem(what, fd, fp)
- else:
- logging.error('Found %s at %s' % (what, fd))
- return True
- return False
+ return ignoreproblem(what, fd, fp)
+ if todelete(fd):
+ return removeproblem(what, fd, fp)
+ logging.error('Found %s at %s' % (what, fd))
+ return 1
+
+ get_mime_type = init_mime_type()
# Iterate through all files in the source code
for r, d, f in os.walk(build_dir, topdown=True):
fp = os.path.join(r, curfile)
fd = fp[len(build_dir) + 1:]
- try:
- mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
- except UnicodeError:
- warnproblem('malformed magic number', fd)
+ mime = get_mime_type(fp)
if mime == 'application/x-sharedlib':
count += handleproblem('shared library', fd, fp)
elif mime == 'application/x-archive':
count += handleproblem('static library', fd, fp)
- elif mime == 'application/x-executable':
+ elif mime == 'application/x-executable' or mime == 'application/x-mach-binary':
count += handleproblem('binary executable', fd, fp)
elif mime == 'application/x-java-applet':
'application/zip',
'application/java-archive',
'application/octet-stream',
- 'binary',
- ):
+ 'binary', ):
if has_extension(fp, 'apk'):
removeproblem('APK file', fd, fp)
warnproblem('unknown compressed or binary file', fd)
elif has_extension(fp, 'java'):
+ if not os.path.isfile(fp):
+ continue
for line in file(fp):
if 'DexClassLoader' in line:
count += handleproblem('DexClassLoader', fd, fp)
break
- if ms is not None:
- ms.close()
+
+ elif has_extension(fp, 'gradle'):
+ if not os.path.isfile(fp):
+ continue
+ for i, line in enumerate(file(fp)):
+ i = i + 1
+ if any(suspect.match(line) for suspect in usual_suspects):
+ count += handleproblem('usual suspect at line %d' % i, fd, fp)
+ break
+
+ for p in scanignore:
+ if p not in scanignore_worked:
+ logging.error('Unused scanignore path: %s' % p)
+ count += 1
+
+ for p in scandelete:
+ if p not in scandelete_worked:
+ logging.error('Unused scandelete path: %s' % p)
+ count += 1
# Presence of a jni directory without buildjni=yes might
# indicate a problem (if it's not a problem, explicitly use
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
- if os.path.exists(self.path):
+ if os.path.isfile(self.path):
for line in file(self.path):
t = line.rstrip().split(' ')
if len(t) == 2:
:param apkfile: full path to the apk to check"""
- p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
- config['build_tools'], 'aapt'),
- 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
+ p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
+ output=False)
if p.returncode != 0:
logging.critical("Failed to get apk manifest information")
sys.exit(1)
class AsynchronousFileReader(threading.Thread):
+
'''
Helper class to implement asynchronous reading of a file
in a separate thread. Pushes read lines on a queue to
output = ''
-def SilentPopen(commands, cwd=None, shell=False):
- return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
+def SdkToolsPopen(commands, cwd=None, output=True):
+ cmd = commands[0]
+ if cmd not in config:
+ config[cmd] = find_sdk_tools_cmd(commands[0])
+ return FDroidPopen([config[cmd]] + commands[1:],
+ cwd=cwd, output=output)
-def FDroidPopen(commands, cwd=None, shell=False, output=True):
+def FDroidPopen(commands, cwd=None, output=True):
"""
Run a command and capture the possibly huge output.
logging.debug("> %s" % ' '.join(commands))
result = PopenResult()
- p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ p = None
+ try:
+ p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ except OSError, e:
+ raise BuildException("OSError while trying to execute " +
+ ' '.join(commands) + ': ' + str(e))
stdout_queue = Queue.Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
re.compile(r'^[\t ]*signingConfig [^ ]*$'),
re.compile(r'.*android\.signingConfigs\.[^{]*$'),
re.compile(r'.*variant\.outputFile = .*'),
+ re.compile(r'.*output\.outputFile = .*'),
re.compile(r'.*\.readLine\(.*'),
- ]
+ ]
for root, dirs, files in os.walk(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
changed = False
opened = 0
+ i = 0
with open(path, "w") as o:
- for line in lines:
+ while i < len(lines):
+ line = lines[i]
+ i += 1
+ while line.endswith('\\\n'):
+ line = line.rstrip('\\\n') + lines[i]
+ i += 1
+
if comment.match(line):
continue
'project.properties',
'build.properties',
'default.properties',
- 'ant.properties',
- ]:
+ 'ant.properties', ]:
if propfile in files:
path = os.path.join(root, propfile)
logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
-def replace_config_vars(cmd):
+def reset_env_path():
+ global env, orig_path
+ env['PATH'] = orig_path
+
+
+def add_to_env_path(path):
+ global env
+ paths = env['PATH'].split(os.pathsep)
+ if path in paths:
+ return
+ paths.append(path)
+ env['PATH'] = os.pathsep.join(paths)
+
+
+def replace_config_vars(cmd, build):
+ global env
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
- cmd = cmd.replace('$$NDK$$', config['ndk_path'])
+ # env['ANDROID_NDK'] is set in build_local right before prepare_source
+ cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
+ if build is not None:
+ cmd = cmd.replace('$$COMMIT$$', build['commit'])
+ cmd = cmd.replace('$$VERSION$$', build['version'])
+ cmd = cmd.replace('$$VERCODE$$', build['vercode'])
return cmd
o.write(line)
if not placed:
o.write('android.library.reference.%d=%s\n' % (number, relpath))
+
+
+def verify_apks(signed_apk, unsigned_apk, tmp_dir):
+ """Verify that two apks are the same
+
+ One of the inputs is signed, the other is unsigned. The signature metadata
+ is transferred from the signed to the unsigned apk, and then jarsigner is
+ used to verify that the signature from the signed apk is also varlid for
+ the unsigned one.
+ :param signed_apk: Path to a signed apk file
+ :param unsigned_apk: Path to an unsigned apk file expected to match it
+ :param tmp_dir: Path to directory for temporary files
+ :returns: None if the verification is successful, otherwise a string
+ describing what went wrong.
+ """
+ sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+ with ZipFile(signed_apk) as signed_apk_as_zip:
+ meta_inf_files = ['META-INF/MANIFEST.MF']
+ for f in signed_apk_as_zip.namelist():
+ if sigfile.match(f):
+ meta_inf_files.append(f)
+ if len(meta_inf_files) < 3:
+ return "Signature files missing from {0}".format(signed_apk)
+ signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
+ with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
+ for meta_inf_file in meta_inf_files:
+ unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
+
+ if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
+ logging.info("...NOT verified - {0}".format(signed_apk))
+ return compare_apks(signed_apk, unsigned_apk, tmp_dir)
+ logging.info("...successfully verified")
+ return None
+
+
+def compare_apks(apk1, apk2, tmp_dir):
+ """Compare two apks
+
+ Returns None if the apk content is the same (apart from the signing key),
+ otherwise a string describing what's different, or what went wrong when
+ trying to do the comparison.
+ """
+
+ badchars = re.compile('''[/ :;'"]''')
+ apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
+ apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
+ for d in [apk1dir, apk2dir]:
+ if os.path.exists(d):
+ shutil.rmtree(d)
+ os.mkdir(d)
+ os.mkdir(os.path.join(d, 'jar-xf'))
+
+ if subprocess.call(['jar', 'xf',
+ os.path.abspath(apk1)],
+ cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
+ return("Failed to unpack " + apk1)
+ if subprocess.call(['jar', 'xf',
+ os.path.abspath(apk2)],
+ cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
+ return("Failed to unpack " + apk2)
+
+ # try to find apktool in the path, if it hasn't been manually configed
+ if 'apktool' not in config:
+ tmp = find_command('apktool')
+ if tmp is not None:
+ config['apktool'] = tmp
+ if 'apktool' in config:
+ if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
+ cwd=apk1dir) != 0:
+ return("Failed to unpack " + apk1)
+ if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
+ cwd=apk2dir) != 0:
+ return("Failed to unpack " + apk2)
+
+ p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
+ lines = p.output.splitlines()
+ if len(lines) != 1 or 'META-INF' not in lines[0]:
+ meld = find_command('meld')
+ if meld is not None:
+ p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
+ return("Unexpected diff output - " + p.output)
+
+ # since everything verifies, delete the comparison to keep cruft down
+ shutil.rmtree(apk1dir)
+ shutil.rmtree(apk2dir)
+
+ # If we get here, it seems like they're the same!
+ return None
+
+
+def find_command(command):
+ '''find the full path of a command, or None if it can't be found in the PATH'''
+
+ def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+ fpath, fname = os.path.split(command)
+ if fpath:
+ if is_exe(command):
+ return command
+ else:
+ for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, command)
+ if is_exe(exe_file):
+ return exe_file
+
+ return None
+
+
+def genpassword():
+ '''generate a random password for when generating keys'''
+ h = hashlib.sha256()
+ h.update(os.urandom(16)) # salt
+ h.update(bytes(socket.getfqdn()))
+ return h.digest().encode('base64').strip()
+
+
+def genkeystore(localconfig):
+ '''Generate a new key with random passwords and add it to new keystore'''
+ logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
+ keystoredir = os.path.dirname(localconfig['keystore'])
+ if keystoredir is None or keystoredir == '':
+ keystoredir = os.path.join(os.getcwd(), keystoredir)
+ if not os.path.exists(keystoredir):
+ os.makedirs(keystoredir, mode=0o700)
+
+ write_password_file("keystorepass", localconfig['keystorepass'])
+ write_password_file("keypass", localconfig['keypass'])
+ p = FDroidPopen(['keytool', '-genkey',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-keyalg', 'RSA', '-keysize', '4096',
+ '-sigalg', 'SHA256withRSA',
+ '-validity', '10000',
+ '-storepass:file', config['keystorepassfile'],
+ '-keypass:file', config['keypassfile'],
+ '-dname', localconfig['keydname']])
+ # TODO keypass should be sent via stdin
+ if p.returncode != 0:
+ raise BuildException("Failed to generate key", p.output)
+ os.chmod(localconfig['keystore'], 0o0600)
+ # now show the lovely key that was just generated
+ p = FDroidPopen(['keytool', '-list', '-v',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:file', config['keystorepassfile']])
+ logging.info(p.output.strip() + '\n\n')
+
+
+def write_to_config(thisconfig, key, value=None):
+ '''write a key/value to the local config.py'''
+ if value is None:
+ origkey = key + '_orig'
+ value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
+ with open('config.py', 'r') as f:
+ data = f.read()
+ pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
+ repl = '\n' + key + ' = "' + value + '"'
+ data = re.sub(pattern, repl, data)
+ # if this key is not in the file, append it
+ if not re.match('\s*' + key + '\s*=\s*"', data):
+ data += repl
+ # make sure the file ends with a carraige return
+ if not re.match('\n$', data):
+ data += '\n'
+ with open('config.py', 'w') as f:
+ f.writelines(data)
+
+
+def parse_xml(path):
+ return XMLElementTree.parse(path).getroot()
+
+
+def string_is_integer(string):
+ try:
+ int(string)
+ return True
+ except ValueError:
+ return False
+
+
+def download_file(url, local_filename=None, dldir='tmp'):
+ filename = url.split('/')[-1]
+ if local_filename is None:
+ local_filename = os.path.join(dldir, filename)
+ # the stream=True parameter keeps memory usage low
+ r = requests.get(url, stream=True)
+ with open(local_filename, 'wb') as f:
+ for chunk in r.iter_content(chunk_size=1024):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ f.flush()
+ return local_filename
sigpath = os.path.join(output_dir, sigfilename)
if not os.path.exists(sigpath):
- p = FDroidPopen(['gpg', '-a',
- '--output', sigpath,
- '--detach-sig',
- os.path.join(output_dir, apkfilename)])
+ gpgargs = ['gpg', '-a',
+ '--output', sigpath,
+ '--detach-sig']
+ if 'gpghome' in config:
+ gpgargs.extend(['--homedir', config['gpghome']])
+ gpgargs.append(os.path.join(output_dir, apkfilename))
+ p = FDroidPopen(gpgargs)
if p.returncode != 0:
logging.error("Signing failed.")
sys.exit(1)
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read()
- # Works for Google Code and BitBucket...
+ # Works for BitBucket
index = page.find('hg clone')
if index != -1:
repotype = 'hg'
repo = repo.split('"')[0]
return (repotype, repo)
- # Works for Google Code and BitBucket...
+ # Works for BitBucket
index = page.find('git clone')
if index != -1:
repotype = 'git'
repo = repo.split('"')[0]
return (repotype, repo)
- # Google Code only...
- index = page.find('svn checkout')
- if index != -1:
- repotype = 'git-svn'
- repo = page[index + 13:]
- prefix = '<strong><em>http</em></strong>'
- if not repo.startswith(prefix):
- return (None, "Unexpected checkout instructions format")
- repo = 'http' + repo[len(prefix):]
- index = repo.find('<')
- if index == -1:
- return (None, "Error while getting repo address - no end tag? '" + repo + "'")
- repo = repo[:index]
- index = repo.find(' ')
- if index == -1:
- return (None, "Error while getting repo address - no space? '" + repo + "'")
- repo = repo[:index]
- repo = repo.split('"')[0]
- return (repotype, repo)
-
return (None, "No information found." + page)
config = None
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
- parser.add_option("-r", "--repo", default=None,
- help="Allows a different repo to be specified for a multi-repo google code project")
parser.add_option("--rev", default=None,
help="Allows a different revision (or git branch) to be specified for the initial import")
(options, args) = parser.parse_args()
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
+ website = ""
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
repo = url
repotype = 'git'
- sourcecode = url
+ sourcecode = url + '/tree/HEAD'
issuetracker = url + '/issues'
- elif url.startswith('https://gitorious.org/'):
- projecttype = 'gitorious'
- repo = 'https://git.gitorious.org/' + url[22:] + '.git'
- repotype = 'git'
- sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
- elif (url.startswith('http://code.google.com/p/') or
- url.startswith('https://code.google.com/p/')):
- if not url.endswith('/'):
- url += '/'
- projecttype = 'googlecode'
- sourcecode = url + 'source/checkout'
- if options.repo:
- sourcecode += "?repo=" + options.repo
- issuetracker = url + 'issues/list'
-
- # Figure out the repo type and adddress...
- repotype, repo = getrepofrompage(sourcecode)
- if not repotype:
- logging.error("Unable to determine vcs type. " + repo)
- sys.exit(1)
-
- # Figure out the license...
- req = urllib.urlopen(url)
- if req.getcode() != 200:
- logging.error('Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()))
- sys.exit(1)
- page = req.read()
- index = page.find('Code license')
- if index == -1:
- logging.error("Couldn't find license data")
- sys.exit(1)
- ltext = page[index:]
- lprefix = 'rel="nofollow">'
- index = ltext.find(lprefix)
- if index == -1:
- logging.error("Couldn't find license text")
- sys.exit(1)
- ltext = ltext[index + len(lprefix):]
- index = ltext.find('<')
- if index == -1:
- logging.error("License text not formatted as expected")
- sys.exit(1)
- ltext = ltext[:index]
- if ltext == 'GNU GPL v3':
- license = 'GPLv3'
- elif ltext == 'GNU GPL v2':
- license = 'GPLv2'
- elif ltext == 'Apache License 2.0':
- license = 'Apache2'
- elif ltext == 'MIT License':
- license = 'MIT'
- elif ltext == 'GNU Lesser GPL':
- license = 'LGPL'
- elif ltext == 'Mozilla Public License 1.1':
- license = 'MPL'
- elif ltext == 'New BSD License':
- license = 'NewBSD'
- else:
- logging.error("License " + ltext + " is not recognised")
- sys.exit(1)
-
if not projecttype:
logging.error("Unable to determine the project type.")
logging.error("The URL you supplied was not in one of the supported formats. Please consult")
logging.error("the manual for a list of supported formats, and supply one of those.")
sys.exit(1)
+ # Ensure we have a sensible-looking repo address at this point. If not, we
+ # might have got a page format we weren't expecting. (Note that we
+ # specifically don't want git@...)
+ if ((repotype != 'bzr' and (not repo.startswith('http://') and
+ not repo.startswith('https://') and
+ not repo.startswith('git://'))) or
+ ' ' in repo):
+ logging.error("Repo address '{0}' does not seem to be valid".format(repo))
+ sys.exit(1)
+
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
src_dir = os.path.join(tmp_dir, 'importer')
root_dir = src_dir
# Extract some information...
- paths = common.manifest_paths(root_dir, None)
+ paths = common.manifest_paths(root_dir, [])
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import glob
-import hashlib
import os
import re
import shutil
import logging
import common
-from common import FDroidPopen, BuildException
config = {}
options = None
-def write_to_config(key, value):
- '''write a key/value to the local config.py'''
- with open('config.py', 'r') as f:
- data = f.read()
- pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
- repl = '\n' + key + ' = "' + value + '"'
- data = re.sub(pattern, repl, data)
- with open('config.py', 'w') as f:
- f.writelines(data)
-
-
def disable_in_config(key, value):
'''write a key/value to the local config.py, then comment it out'''
with open('config.py', 'r') as f:
f.writelines(data)
-def genpassword():
- '''generate a random password for when generating keys'''
- h = hashlib.sha256()
- h.update(os.urandom(16)) # salt
- h.update(bytes(socket.getfqdn()))
- return h.digest().encode('base64').strip()
-
-
-def genkey(keystore, repo_keyalias, password, keydname):
- '''generate a new keystore with a new key in it for signing repos'''
- logging.info('Generating a new key in "' + keystore + '"...')
- common.write_password_file("keystorepass", password)
- common.write_password_file("keypass", password)
- p = FDroidPopen(['keytool', '-genkey',
- '-keystore', keystore, '-alias', repo_keyalias,
- '-keyalg', 'RSA', '-keysize', '4096',
- '-sigalg', 'SHA256withRSA',
- '-validity', '10000',
- '-storepass:file', config['keystorepassfile'],
- '-keypass:file', config['keypassfile'],
- '-dname', keydname])
- # TODO keypass should be sent via stdin
- if p.returncode != 0:
- raise BuildException("Failed to generate key", p.output)
- # now show the lovely key that was just generated
- p = FDroidPopen(['keytool', '-list', '-v',
- '-keystore', keystore, '-alias', repo_keyalias,
- '-storepass:file', config['keystorepassfile']])
- logging.info(p.output.strip() + '\n\n')
-
-
def main():
global options, config
# find root install prefix
tmp = os.path.dirname(sys.argv[0])
if os.path.basename(tmp) == 'bin':
- prefix = os.path.dirname(tmp)
- examplesdir = prefix + '/share/doc/fdroidserver/examples'
+ prefix = None
+ egg_link = os.path.join(tmp, '..', 'local/lib/python2.7/site-packages/fdroidserver.egg-link')
+ if os.path.exists(egg_link):
+ # installed from local git repo
+ examplesdir = os.path.join(open(egg_link).readline().rstrip(), 'examples')
+ else:
+ prefix = os.path.dirname(os.path.dirname(__file__)) # use .egg layout
+ if not prefix.endswith('.egg'): # use UNIX layout
+ prefix = os.path.dirname(tmp)
+ examplesdir = prefix + '/share/doc/fdroidserver/examples'
else:
# we're running straight out of the git repo
prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
examplesdir = prefix + '/examples'
+ aapt = None
fdroiddir = os.getcwd()
- test_config = common.get_default_config()
+ test_config = dict()
+ common.fill_config_defaults(test_config)
# track down where the Android SDK is, the default is to use the path set
# in ANDROID_HOME if that exists, otherwise None
if options.android_home is not None:
test_config['sdk_path'] = options.android_home
elif not common.test_sdk_exists(test_config):
- # if neither --android-home nor the default sdk_path exist, prompt the user
- default_sdk_path = '/opt/android-sdk'
- while not options.no_prompt:
- try:
- s = raw_input('Enter the path to the Android SDK ('
- + default_sdk_path + ') here:\n> ')
- except KeyboardInterrupt:
- print('')
- sys.exit(1)
- if re.match('^\s*$', s) is not None:
- test_config['sdk_path'] = default_sdk_path
- else:
- test_config['sdk_path'] = s
- if common.test_sdk_exists(test_config):
- break
+ if os.path.isfile('/usr/bin/aapt'):
+ # remove sdk_path and build_tools, they are not required
+ test_config.pop('sdk_path', None)
+ test_config.pop('build_tools', None)
+ # make sure at least aapt is found, since this can't do anything without it
+ test_config['aapt'] = common.find_sdk_tools_cmd('aapt')
+ else:
+ # if neither --android-home nor the default sdk_path exist, prompt the user
+ default_sdk_path = '/opt/android-sdk'
+ while not options.no_prompt:
+ try:
+ s = raw_input('Enter the path to the Android SDK ('
+ + default_sdk_path + ') here:\n> ')
+ except KeyboardInterrupt:
+ print('')
+ sys.exit(1)
+ if re.match('^\s*$', s) is not None:
+ test_config['sdk_path'] = default_sdk_path
+ else:
+ test_config['sdk_path'] = s
+ if common.test_sdk_exists(test_config):
+ break
if not common.test_sdk_exists(test_config):
sys.exit(3)
shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py')
os.chmod('config.py', 0o0600)
- write_to_config('sdk_path', test_config['sdk_path'])
+ # If android_home is None, test_config['sdk_path'] will be used and
+ # "$ANDROID_HOME" may be used if the env var is set up correctly.
+ # If android_home is not None, the path given from the command line
+ # will be directly written in the config.
+ if 'sdk_path' in test_config:
+ common.write_to_config(test_config, 'sdk_path', options.android_home)
else:
logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
logging.info('Try running `fdroid init` in an empty directory.')
sys.exit()
- # try to find a working aapt, in all the recent possible paths
- build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
- aaptdirs = []
- aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
- aaptdirs.append(build_tools)
- for f in os.listdir(build_tools):
- if os.path.isdir(os.path.join(build_tools, f)):
- aaptdirs.append(os.path.join(build_tools, f))
- for d in sorted(aaptdirs, reverse=True):
- if os.path.isfile(os.path.join(d, 'aapt')):
- aapt = os.path.join(d, 'aapt')
- break
- if os.path.isfile(aapt):
- dirname = os.path.basename(os.path.dirname(aapt))
- if dirname == 'build-tools':
- # this is the old layout, before versioned build-tools
- test_config['build_tools'] = ''
- else:
- test_config['build_tools'] = dirname
- write_to_config('build_tools', test_config['build_tools'])
- if not common.test_build_tools_exists(test_config):
- sys.exit(3)
+ if 'aapt' not in test_config or not os.path.isfile(test_config['aapt']):
+ # try to find a working aapt, in all the recent possible paths
+ build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
+ aaptdirs = []
+ aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
+ aaptdirs.append(build_tools)
+ for f in os.listdir(build_tools):
+ if os.path.isdir(os.path.join(build_tools, f)):
+ aaptdirs.append(os.path.join(build_tools, f))
+ for d in sorted(aaptdirs, reverse=True):
+ if os.path.isfile(os.path.join(d, 'aapt')):
+ aapt = os.path.join(d, 'aapt')
+ break
+ if os.path.isfile(aapt):
+ dirname = os.path.basename(os.path.dirname(aapt))
+ if dirname == 'build-tools':
+ # this is the old layout, before versioned build-tools
+ test_config['build_tools'] = ''
+ else:
+ test_config['build_tools'] = dirname
+ common.write_to_config(test_config, 'build_tools')
+ common.ensure_build_tools_exists(test_config)
# now that we have a local config.py, read configuration...
config = common.read_config(options)
- # track down where the Android NDK is
- ndk_path = '/opt/android-ndk'
- if os.path.isdir(config['ndk_path']):
- ndk_path = config['ndk_path']
- elif 'ANDROID_NDK' in os.environ.keys():
- logging.info('using ANDROID_NDK')
- ndk_path = os.environ['ANDROID_NDK']
- if os.path.isdir(ndk_path):
- write_to_config('ndk_path', ndk_path)
- # the NDK is optional so we don't prompt the user for it if its not found
+ # the NDK is optional and there may be multiple versions of it, so it's
+ # left for the user to configure
# find or generate the keystore for the repo signing key. First try the
# path written in the default config.py. Then check if the user has
if not os.path.exists(keystore):
logging.info('"' + keystore
+ '" does not exist, creating a new keystore there.')
- write_to_config('keystore', keystore)
+ common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
- write_to_config('repo_keyalias', repo_keyalias)
+ common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
if options.distinguished_name:
keydname = options.distinguished_name
- write_to_config('keydname', keydname)
+ common.write_to_config(test_config, 'keydname', keydname)
if keystore == 'NONE': # we're using a smartcard
- write_to_config('repo_keyalias', '1') # seems to be the default
+ common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
disable_in_config('keypass', 'never used with smartcard')
- write_to_config('smartcardoptions',
- ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
- + '-providerClass sun.security.pkcs11.SunPKCS11 '
- + '-providerArg opensc-fdroid.cfg'))
+ common.write_to_config(test_config, 'smartcardoptions',
+ ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+ + '-providerClass sun.security.pkcs11.SunPKCS11 '
+ + '-providerArg opensc-fdroid.cfg'))
# find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'):
with open('opensc-fdroid.cfg', 'w') as f:
f.write(opensc_fdroid)
elif not os.path.exists(keystore):
- # no existing or specified keystore, generate the whole thing
- keystoredir = os.path.dirname(keystore)
- if not os.path.exists(keystoredir):
- os.makedirs(keystoredir, mode=0o700)
- password = genpassword()
- write_to_config('keystorepass', password)
- write_to_config('keypass', password)
- if options.repo_keyalias is None:
- repo_keyalias = socket.getfqdn()
- write_to_config('repo_keyalias', repo_keyalias)
- if not options.distinguished_name:
- keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
- write_to_config('keydname', keydname)
- genkey(keystore, repo_keyalias, password, keydname)
+ password = common.genpassword()
+ c = dict(test_config)
+ c['keystorepass'] = password
+ c['keypass'] = password
+ c['repo_keyalias'] = socket.getfqdn()
+ c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid'
+ common.write_to_config(test_config, 'keystorepass', password)
+ common.write_to_config(test_config, 'keypass', password)
+ common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias'])
+ common.write_to_config(test_config, 'keydname', c['keydname'])
+ common.genkeystore(c)
logging.info('Built repo based in "' + fdroiddir + '"')
logging.info('with this config:')
logging.info(' Android SDK:\t\t\t' + config['sdk_path'])
- logging.info(' Android SDK Build Tools:\t' + os.path.dirname(aapt))
- logging.info(' Android NDK (optional):\t' + ndk_path)
+ if aapt:
+ logging.info(' Android SDK Build Tools:\t' + os.path.dirname(aapt))
+ logging.info(' Android NDK r10e (optional):\t$ANDROID_NDK')
logging.info(' Keystore for signing key:\t' + keystore)
if repo_keyalias is not None:
logging.info(' Alias for key in store:\t' + repo_keyalias)
import logging
import common
-from common import FDroidPopen, FDroidException
+from common import SdkToolsPopen, FDroidException
options = None
config = None
def devices():
- p = FDroidPopen([config['adb'], "devices"])
+ p = SdkToolsPopen(['adb', "devices"])
if p.returncode != 0:
raise FDroidException("An error occured when finding devices: %s" % p.output)
lines = p.output.splitlines()
logging.info("Installing %s..." % apk)
for dev in devs:
logging.info("Installing %s on %s..." % (apk, dev))
- p = FDroidPopen([config['adb'], "-s", dev, "install", apk])
+ p = SdkToolsPopen(['adb', "-s", dev, "install", apk])
fail = ""
for line in p.output.splitlines():
if line.startswith("Failure"):
import logging
import common
import metadata
+import sys
from collections import Counter
+from sets import Set
config = None
options = None
+
+def enforce_https(domain):
+ return (re.compile(r'.*[^sS]://[^/]*' + re.escape(domain) + r'(/.*)?'),
+ domain + " URLs should always use https://")
+
+https_enforcings = [
+ enforce_https('github.com'),
+ enforce_https('gitlab.com'),
+ enforce_https('gitorious.org'),
+ enforce_https('apache.org'),
+ enforce_https('google.com'),
+ enforce_https('svn.code.sf.net'),
+ enforce_https('googlecode.com'),
+]
+
+
+def forbid_shortener(domain):
+ return (re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'),
+ "URL shorteners should not be used")
+
+http_url_shorteners = [
+ forbid_shortener('goo.gl'),
+ forbid_shortener('t.co'),
+ forbid_shortener('ur1.ca'),
+]
+
+http_warnings = https_enforcings + http_url_shorteners + [
+ (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
+ "Appending .git is not necessary"),
+ (re.compile(r'(.*/blob/master/|.*raw\.github.com/[^/]*/[^/]*/master/)'),
+ "Use /HEAD/ instead of /master/ to point at a file in the default branch"),
+ # TODO enable in August 2015, when Google Code goes read-only
+ # (re.compile(r'.*://code\.google\.com/.*'),
+ # "code.google.com will be soon switching down, perhaps the project moved to github.com?"),
+]
+
regex_warnings = {
- 'Web Site': [
- (re.compile(r'.*[^sS]://github\.com/.*'),
- "github URLs should always use https:// not http://"),
- (re.compile(r'.*[^sS]://code\.google\.com/.*'),
- "code.google.com URLs should always use https:// not http://"),
- ],
- 'Source Code': [
- (re.compile(r'.*[^sS]://github\.com/.*'),
- "github URLs should always use https:// (not http://, git://, or git@)"),
- (re.compile(r'.*code\.google\.com/p/[^/]+[/]*$'),
- "/source is missing"),
- (re.compile(r'.*[^sS]://code\.google\.com/.*'),
- "code.google.com URLs should always use https:// not http://"),
- (re.compile(r'.*[^sS]://dl\.google\.com/.*'),
- "dl.google.com URLs should always use https:// not http://"),
- (re.compile(r'.*[^sS]://gitorious\.org/.*'),
- "gitorious URLs should always use https:// (not http://, git://, or git@)"),
- ],
- 'Repo': [
- (re.compile(r'.*[^sS]://code\.google\.com/.*'),
- "code.google.com URLs should always use https:// not http://"),
- (re.compile(r'.*[^sS]://dl\.google\.com/.*'),
- "dl.google.com URLs should always use https:// not http://"),
- (re.compile(r'.*[^sS]://github\.com/.*'),
- "github URLs should always use https:// (not http://, git://, or git@)"),
- (re.compile(r'.*[^sS]://gitorious\.org/.*'),
- "gitorious URLs should always use https:// (not http://, git://, or git@)"),
- (re.compile(r'.*[^sS]://[^.]*\.googlecode\.com/svn/?.*'),
- "Google Code SVN URLs should always use https:// (not http:// or svn://)"),
- (re.compile(r'.*[^sS]://svn\.apache\.org/repos/?.*'),
- "Apache SVN URLs should always use https:// (not http:// or svn://)"),
- (re.compile(r'.*[^sS]://svn\.code\.sf\.net/.*'),
- "Sourceforge SVN URLs should always use https:// (not http:// or svn://)"),
- ],
- 'Issue Tracker': [
- (re.compile(r'.*code\.google\.com/p/[^/]+[/]*$'),
- "/issues is missing"),
- (re.compile(r'.*[^sS]://code\.google\.com/.*'),
- "code.google.com URLs should always use https:// not http://"),
+ 'Web Site': http_warnings + [
+ ],
+ 'Source Code': http_warnings + [
+ ],
+ 'Repo': https_enforcings + [
+ ],
+ 'Issue Tracker': http_warnings + [
(re.compile(r'.*github\.com/[^/]+/[^/]+[/]*$'),
"/issues is missing"),
- (re.compile(r'.*[^sS]://github\.com/.*'),
- "github URLs should always use https:// not http://"),
- (re.compile(r'.*[^sS]://gitorious\.org/.*'),
- "gitorious URLs should always use https:// not http://"),
- ],
+ ],
+ 'Donate': http_warnings + [
+ (re.compile(r'.*flattr\.com'),
+ "Flattr donation methods belong in the FlattrID flag"),
+ ],
+ 'Changelog': http_warnings + [
+ ],
'License': [
(re.compile(r'^(|None|Unknown)$'),
"No license specified"),
- ],
+ ],
'Summary': [
(re.compile(r'^$'),
"Summary yet to be filled"),
- ],
+ (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE),
+ "No need to specify that the app is Free Software"),
+ (re.compile(r'.*((your|for).*android|android.*(app|device|client|port|version))', re.IGNORECASE),
+ "No need to specify that the app is for Android"),
+ (re.compile(r'.*[a-z0-9][.!?]( |$)'),
+ "Punctuation should be avoided"),
+ ],
'Description': [
(re.compile(r'^No description available$'),
"Description yet to be filled"),
- (re.compile(r'[ ]*[*#][^ .]'),
+ (re.compile(r'\s*[*#][^ .]'),
"Invalid bulleted list"),
- (re.compile(r'^ '),
+ (re.compile(r'^\s'),
"Unnecessary leading space"),
- ],
+ (re.compile(r'.*\s$'),
+ "Unnecessary trailing space"),
+ ],
}
-regex_pedantic = {
- 'Web Site': [
- (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
- "Appending .git is not necessary"),
- (re.compile(r'.*code\.google\.com/p/[^/]+/[^w]'),
- "Possible incorrect path appended to google code project site"),
- ],
- 'Source Code': [
- (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
- "Appending .git is not necessary"),
- (re.compile(r'.*code\.google\.com/p/[^/]+/source/.*'),
- "/source is often enough on its own"),
- ],
- 'Repo': [
- (re.compile(r'^http://.*'),
- "use https:// if available"),
- (re.compile(r'^svn://.*'),
- "use https:// if available"),
- ],
- 'Issue Tracker': [
- (re.compile(r'.*code\.google\.com/p/[^/]+/issues/.*'),
- "/issues is often enough on its own"),
- (re.compile(r'.*github\.com/[^/]+/[^/]+/issues/.*'),
- "/issues is often enough on its own"),
- ],
- 'Summary': [
- (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE),
- "No need to specify that the app is Free Software"),
- (re.compile(r'.*[a-z0-9][.,!?][ $]'),
- "Punctuation should be avoided"),
- ],
- }
+categories = Set([
+ "Children",
+ "Development",
+ "Games",
+ "Internet",
+ "Multimedia",
+ "Navigation",
+ "Office",
+ "Phone & SMS",
+ "Reading",
+ "Science & Education",
+ "Security",
+ "System",
+ "Wallpaper",
+])
+
+desc_url = re.compile("[^[]\[([^ ]+)( |\]|$)")
def main():
print ' %s' % message
count['warn'] += 1
- def pwarn(message):
- if options.pedantic:
- warn(message)
-
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
- parser.add_option("-p", "--pedantic", action="store_true", default=False,
- help="Show pedantic warnings that might give false positives")
(options, args) = parser.parse_args()
config = common.read_config(options)
allapps = metadata.read_metadata(xref=False)
apps = common.read_app_args(args, allapps, False)
- for appid, app in apps.iteritems():
- curid = appid
- lastcommit = ''
+ filling_ucms = re.compile('^(Tags.*|RepoManifest.*)')
+ for appid, app in apps.iteritems():
if app['Disabled']:
continue
+ curid = appid
+ count['app_total'] += 1
+
+ # enabled_builds = 0
+ lowest_vercode = -1
+ curbuild = None
for build in app['builds']:
- if build['commit'] and not build['disable']:
- lastcommit = build['commit']
+ if not build['disable']:
+ # enabled_builds += 1
+ vercode = int(build['vercode'])
+ if lowest_vercode == -1 or vercode < lowest_vercode:
+ lowest_vercode = vercode
+ if not curbuild or int(build['vercode']) > int(curbuild['vercode']):
+ curbuild = build
- # Potentially incorrect UCM
- if (app['Update Check Mode'] == 'RepoManifest' and
- any(s in lastcommit for s in '.,_-/')):
- pwarn("Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (
- lastcommit, app['Update Check Mode']))
+ # Incorrect UCM
+ if (curbuild and curbuild['commit']
+ and app['Update Check Mode'] == 'RepoManifest'
+ and not curbuild['commit'].startswith('unknown')
+ and curbuild['vercode'] == app['Current Version Code']
+ and not curbuild['forcevercode']
+ and any(s in curbuild['commit'] for s in '.,_-/')):
+ warn("Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (
+ curbuild['commit'], app['Update Check Mode']))
# Summary size limit
summ_chars = len(app['Summary'])
if app['Web Site'] and app['Source Code']:
if app['Web Site'].lower() == app['Source Code'].lower():
warn("Website '%s' is just the app's source code link" % app['Web Site'])
- app['Web Site'] = ''
+
+ if filling_ucms.match(app['Update Check Mode']):
+ if all(app[f] == metadata.app_defaults[f] for f in [
+ 'Auto Name',
+ 'Current Version',
+ 'Current Version Code',
+ ]):
+ warn("UCM is set but it looks like checkupdates hasn't been run yet")
+
+ if app['Update Check Name'] == appid:
+ warn("Update Check Name is set to the known app id - it can be removed")
+
+ cvc = int(app['Current Version Code'])
+ if cvc > 0 and cvc < lowest_vercode:
+ warn("Current Version Code is lower than any enabled build")
+
+ # Missing or incorrect categories
+ if not app['Categories']:
+ warn("Categories are not set")
+ for categ in app['Categories']:
+ if categ not in categories:
+ warn("Category '%s' is not valid" % categ)
+
+ if app['Name'] and app['Name'] == app['Auto Name']:
+ warn("Name '%s' is just the auto name" % app['Name'])
name = app['Name'] or app['Auto Name']
if app['Summary'] and name:
if app['Summary'].lower() == name.lower():
warn("Summary '%s' is just the app's name" % app['Summary'])
- if app['Summary'] and app['Description'] and len(app['Description']) == 1:
- if app['Summary'].lower() == app['Description'][0].lower():
+ desc = app['Description']
+ if app['Summary'] and desc and len(desc) == 1:
+ if app['Summary'].lower() == desc[0].lower():
warn("Description '%s' is just the app's summary" % app['Summary'])
# Description size limit
- desc_chars = sum(len(l) for l in app['Description'])
- if desc_chars > config['char_limits']['Description']:
+ desc_charcount = sum(len(l) for l in desc)
+ if desc_charcount > config['char_limits']['Description']:
warn("Description of length %s is over the %i char limit" % (
- desc_chars, config['char_limits']['Description']))
+ desc_charcount, config['char_limits']['Description']))
+
+ if (not desc[0] or not desc[-1]
+ or any(not desc[l - 1] and not desc[l] for l in range(1, len(desc)))):
+ warn("Description has an extra empty line")
+
+ # Check for lists using the wrong characters
+ validchars = ['*', '#']
+ lchar = ''
+ lcount = 0
+ for l in app['Description']:
+ if len(l) < 1:
+ continue
+
+ for um in desc_url.finditer(l):
+ url = um.group(1)
+ for m, r in http_warnings:
+ if m.match(url):
+ warn("URL '%s' in Description: %s" % (url, r))
+
+ c = l.decode('utf-8')[0]
+ if c == lchar:
+ lcount += 1
+ if lcount > 3 and lchar not in validchars:
+ warn("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)" % lchar)
+ break
+ else:
+ lchar = c
+ lcount = 1
# Regex checks in all kinds of fields
for f in regex_warnings:
for m, r in regex_warnings[f]:
- t = metadata.metafieldtype(f)
- if t == 'string':
- if m.match(app[f]):
- warn("%s '%s': %s" % (f, app[f], r))
- elif t == 'multiline':
- for l in app[f]:
+ v = app[f]
+ if type(v) == str:
+ if v is None:
+ continue
+ if m.match(v):
+ warn("%s '%s': %s" % (f, v, r))
+ elif type(v) == list:
+ for l in v:
if m.match(l):
warn("%s at line '%s': %s" % (f, l, r))
- # Regex pedantic checks in all kinds of fields
- if options.pedantic:
- for f in regex_pedantic:
- for m, r in regex_pedantic[f]:
- if m.match(app[f]):
- warn("%s '%s': %s" % (f, app[f], r))
-
# Build warnings
for build in app['builds']:
if build['disable']:
if ref.startswith(s):
warn("Branch '%s' used as commit in srclib '%s'" % (
s, srclib))
- for s in ['git clone', 'git svn clone', 'svn checkout', 'svn co', 'hg clone']:
- for flag in ['init', 'prebuild', 'build']:
- if not build[flag]:
- continue
- if s in build[flag]:
- # TODO: This should not be pedantic!
- pwarn("'%s' used in %s '%s'" % (s, flag, build[flag]))
if not curid:
print
- logging.info("Found a total of %i warnings in %i apps." % (count['warn'], count['app']))
+ logging.info("Found a total of %i warnings in %i apps out of %i total." % (
+ count['warn'], count['app'], count['app_total']))
+
+ sys.exit(1 if count['warn'] > 0 else 0)
if __name__ == "__main__":
main()
from collections import OrderedDict
+import common
+
srclibs = None
class MetaDataException(Exception):
+
def __init__(self, value):
self.value = value
('Web Site', ''),
('Source Code', ''),
('Issue Tracker', ''),
+ ('Changelog', ''),
('Donate', None),
('FlattrID', None),
('Bitcoin', None),
('Requires Root', False),
('Repo Type', ''),
('Repo', ''),
+ ('Binaries', None),
('Maintainer Notes', []),
('Archive Policy', None),
('Auto Update Mode', 'None'),
('Current Version', ''),
('Current Version Code', '0'),
('No Source Since', ''),
- ])
+])
# In the order in which they are laid out on files
('scandelete', []),
('build', ''),
('buildjni', []),
+ ('ndk', 'r10e'), # defaults to latest
('preassemble', []),
- ('antcommand', None),
+ ('gradleprops', []),
+ ('antcommands', None),
('novcheck', False),
- ])
+])
# Designates a metadata field type and checks that it matches
FieldValidator("HTTP link",
r'^http[s]?://', None,
- ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
+ ["Web Site", "Source Code", "Issue Tracker", "Changelog", "Donate"], []),
FieldValidator("Bitcoin address",
r'^[a-zA-Z0-9]{27,34}$', None,
["Repo Type"],
[]),
+ FieldValidator("Binaries",
+ r'^http[s]?://', None,
+ ["Binaries"],
+ []),
+
FieldValidator("Archive Policy",
r'^[0-9]+ versions$', None,
["Archive Policy"],
r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
["Update Check Mode"],
[])
- }
+}
# Check an app's metadata information for integrity errors
# Formatter for descriptions. Create an instance, and call parseline() with
# each line of the description source from the metadata. At the end, call
-# end() and then text_plain, text_wiki and text_html will contain the result.
+# end() and then text_wiki and text_html will contain the result.
class DescriptionFormatter:
stNONE = 0
stPARA = 1
bold = False
ital = False
state = stNONE
- text_plain = ''
text_wiki = ''
text_html = ''
linkResolver = None
self.endol()
def endpara(self):
- self.text_plain += '\n'
self.text_html += '</p>'
self.state = self.stNONE
def addtext(self, txt):
p, h = self.linkify(txt)
- self.text_plain += p
self.text_html += h
def parseline(self, line):
self.text_html += '<ul>'
self.state = self.stUL
self.text_html += '<li>'
- self.text_plain += '* '
self.addtext(line[1:])
self.text_html += '</li>'
elif line.startswith('# '):
self.text_html += '<ol>'
self.state = self.stOL
self.text_html += '<li>'
- self.text_plain += '* ' # TODO: lazy - put the numbers in!
self.addtext(line[1:])
self.text_html += '</li>'
else:
self.state = self.stPARA
elif self.state == self.stPARA:
self.text_html += ' '
- self.text_plain += ' '
self.addtext(line)
def end(self):
self.endcur()
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in plain text format.
-def description_plain(lines, linkres):
- ps = DescriptionFormatter(linkres)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_plain
-
-
# Parse multiple lines of description as written in a metadata file, returning
# a single string in wiki format. Used for the Maintainer Notes field as well,
# because it's the same format.
thisinfo['Repo'] = ''
thisinfo['Subdir'] = None
thisinfo['Prepare'] = None
- thisinfo['Srclibs'] = None
if metafile is None:
return thisinfo
def flagtype(name):
- if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
- 'update', 'scanignore', 'scandelete']:
+ if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
+ 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
+ 'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
continue
build[flag] = value
build['type'] = get_build_type()
+ build['ndk_path'] = common.get_ndk_path(build['ndk'])
+
+
+def split_list_values(s):
+ # Port legacy ';' separators
+ l = [v.strip() for v in s.replace(';', ',').split(',')]
+ return [v for v in l if v]
# Parse metadata for a single application.
linedesc = None
def add_buildflag(p, thisbuild):
+ if not p.strip():
+ raise MetaDataException("Empty build flag at {1}"
+ .format(buildlines[0], linedesc))
bv = p.split('=', 1)
if len(bv) != 2:
raise MetaDataException("Invalid build flag at {0} in {1}"
.format(p, linedesc))
t = flagtype(pk)
if t == 'list':
- # Port legacy ';' separators
- thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
+ pv = split_list_values(pv)
+ if pk == 'gradle':
+ if len(pv) == 1 and pv[0] in ['main', 'yes']:
+ pv = ['yes']
+ thisbuild[pk] = pv
elif t == 'string' or t == 'script':
thisbuild[pk] = pv
elif t == 'bool':
elif fieldtype == 'string':
thisinfo[field] = value
elif fieldtype == 'list':
- thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
+ thisinfo[field] = split_list_values(value)
elif fieldtype == 'build':
if value.endswith("\\"):
mode = 2
writefield('Web Site')
writefield('Source Code')
writefield('Issue Tracker')
+ writefield_nonempty('Changelog')
writefield_nonempty('Donate')
writefield_nonempty('FlattrID')
writefield_nonempty('Bitcoin')
if app['Repo Type']:
writefield('Repo Type')
writefield('Repo')
+ if app['Binaries']:
+ writefield('Binaries')
mf.write('\n')
for build in app['builds']:
import common
import metadata
-from common import FDroidPopen, BuildException
+from common import FDroidPopen, SdkToolsPopen, BuildException
config = None
options = None
continue
logging.info("Processing " + apkfile)
- # Figure out the key alias name we'll use. Only the first 8
- # characters are significant, so we'll use the first 8 from
- # the MD5 of the app's ID and hope there are no collisions.
- # If a collision does occur later, we're going to have to
- # come up with a new alogrithm, AND rename all existing keys
- # in the keystore!
- if appid in config['keyaliases']:
- # For this particular app, the key alias is overridden...
- keyalias = config['keyaliases'][appid]
- if keyalias.startswith('@'):
+ # There ought to be valid metadata for this app, otherwise why are we
+ # trying to publish it?
+ if appid not in allapps:
+ logging.error("Unexpected {0} found in unsigned directory"
+ .format(apkfilename))
+ sys.exit(1)
+ app = allapps[appid]
+
+ if app.get('Binaries', None):
+
+ # It's an app where we build from source, and verify the apk
+ # contents against a developer's binary, and then publish their
+ # version if everything checks out.
+ # The binary should already have been retrieved during the build
+ # process.
+ srcapk = apkfile + ".binary"
+
+ # Compare our unsigned one with the downloaded one...
+ compare_result = common.verify_apks(srcapk, apkfile, tmp_dir)
+ if compare_result:
+ logging.error("...verification failed - publish skipped : "
+ + compare_result)
+ continue
+
+ # Success! So move the downloaded file to the repo, and remove
+ # our built version.
+ shutil.move(srcapk, os.path.join(output_dir, apkfilename))
+ os.remove(apkfile)
+
+ else:
+
+ # It's a 'normal' app, i.e. we sign and publish it...
+
+ # Figure out the key alias name we'll use. Only the first 8
+ # characters are significant, so we'll use the first 8 from
+ # the MD5 of the app's ID and hope there are no collisions.
+ # If a collision does occur later, we're going to have to
+ # come up with a new alogrithm, AND rename all existing keys
+ # in the keystore!
+ if appid in config['keyaliases']:
+ # For this particular app, the key alias is overridden...
+ keyalias = config['keyaliases'][appid]
+ if keyalias.startswith('@'):
+ m = md5.new()
+ m.update(keyalias[1:])
+ keyalias = m.hexdigest()[:8]
+ else:
m = md5.new()
- m.update(keyalias[1:])
+ m.update(appid)
keyalias = m.hexdigest()[:8]
- else:
- m = md5.new()
- m.update(appid)
- keyalias = m.hexdigest()[:8]
- logging.info("Key alias: " + keyalias)
-
- # See if we already have a key for this application, and
- # if not generate one...
- p = FDroidPopen(['keytool', '-list',
- '-alias', keyalias, '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile']])
- if p.returncode != 0:
- logging.info("Key does not exist - generating...")
- p = FDroidPopen(['keytool', '-genkey',
- '-keystore', config['keystore'],
- '-alias', keyalias,
- '-keyalg', 'RSA', '-keysize', '2048',
- '-validity', '10000',
+ logging.info("Key alias: " + keyalias)
+
+ # See if we already have a key for this application, and
+ # if not generate one...
+ p = FDroidPopen(['keytool', '-list',
+ '-alias', keyalias, '-keystore', config['keystore'],
+ '-storepass:file', config['keystorepassfile']])
+ if p.returncode != 0:
+ logging.info("Key does not exist - generating...")
+ p = FDroidPopen(['keytool', '-genkey',
+ '-keystore', config['keystore'],
+ '-alias', keyalias,
+ '-keyalg', 'RSA', '-keysize', '2048',
+ '-validity', '10000',
+ '-storepass:file', config['keystorepassfile'],
+ '-keypass:file', config['keypassfile'],
+ '-dname', config['keydname']])
+ # TODO keypass should be sent via stdin
+ if p.returncode != 0:
+ raise BuildException("Failed to generate key")
+
+ # Sign the application...
+ p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile'],
- '-keypass:file', config['keypassfile'],
- '-dname', config['keydname']])
+ '-keypass:file', config['keypassfile'], '-sigalg',
+ 'MD5withRSA', '-digestalg', 'SHA1',
+ apkfile, keyalias])
# TODO keypass should be sent via stdin
if p.returncode != 0:
- raise BuildException("Failed to generate key")
-
- # Sign the application...
- p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile'],
- '-keypass:file', config['keypassfile'], '-sigalg',
- 'MD5withRSA', '-digestalg', 'SHA1',
- apkfile, keyalias])
- # TODO keypass should be sent via stdin
- if p.returncode != 0:
- raise BuildException("Failed to sign application")
-
- # Zipalign it...
- p = FDroidPopen([config['zipalign'], '-v', '4', apkfile,
- os.path.join(output_dir, apkfilename)])
- if p.returncode != 0:
- raise BuildException("Failed to align application")
- os.remove(apkfile)
+ raise BuildException("Failed to sign application")
+
+ # Zipalign it...
+ p = SdkToolsPopen(['zipalign', '-v', '4', apkfile,
+ os.path.join(output_dir, apkfilename)])
+ if p.returncode != 0:
+ raise BuildException("Failed to align application")
+ os.remove(apkfile)
# Move the source tarball into the output directory...
tarfilename = apkfilename[:-4] + '_src.tar.gz'
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+from optparse import OptionParser
import common
import metadata
def main():
+ parser = OptionParser(usage="Usage: %prog")
+ parser.parse_args()
common.read_config(None)
metadata.read_metadata(xref=True)
allapps = metadata.read_metadata()
apps = common.read_app_args(args, allapps, True)
- problems = []
+ probcount = 0
build_dir = 'build'
if not os.path.isdir(build_dir):
extlib_dir, False)
# Do the scan...
- buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
- for problem in buildprobs:
- problems.append(problem + ' in ' + appid
- + ' ' + thisbuild['version'])
+ count = common.scan_source(build_dir, root_dir, thisbuild)
+ if count > 0:
+ logging.warn('Scanner found %d problems in %s (%s)' % (
+ count, appid, thisbuild['vercode']))
+ probcount += count
except BuildException as be:
- msg = "Could not scan app %s due to BuildException: %s" % (appid, be)
- problems.append(msg)
+ logging.warn("Could not scan app %s due to BuildException: %s" % (
+ appid, be))
+ probcount += 1
except VCSException as vcse:
- msg = "VCS error while scanning app %s: %s" % (appid, vcse)
- problems.append(msg)
+ logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
+ probcount += 1
except Exception:
- msg = "Could not scan app %s due to unknown error: %s" % (appid, traceback.format_exc())
- problems.append(msg)
+ logging.warn("Could not scan app %s due to unknown error: %s" % (
+ appid, traceback.format_exc()))
+ probcount += 1
logging.info("Finished:")
- for problem in problems:
- print problem
- print str(len(problems)) + ' problems.'
+ print "%d app(s) with problems" % probcount
if __name__ == "__main__":
main()
# -*- coding: utf-8 -*-
#
# server.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
+import glob
import hashlib
import os
import paramiko
Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
'''
+ logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
+ + config['awsbucket'] + '"')
+
import libcloud.security
libcloud.security.VERIFY_SSL_CERT = True
from libcloud.storage.types import Provider, ContainerDoesNotExistError
def update_serverwebroot(serverwebroot, repo_section):
- rsyncargs = ['rsync', '--archive', '--delete']
+ # use a checksum comparison for accurate comparisons on different
+ # filesystems, for example, FAT has a low resolution timestamp
+ rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
+ if not options.no_checksum:
+ rsyncargs.append('--checksum')
if options.verbose:
rsyncargs += ['--verbose']
if options.quiet:
rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
indexxml = os.path.join(repo_section, 'index.xml')
indexjar = os.path.join(repo_section, 'index.jar')
- # serverwebroot is guaranteed to have a trailing slash in common.py
+ # Upload the first time without the index files and delay the deletion as
+ # much as possible, that keeps the repo functional while this update is
+ # running. Then once it is complete, rerun the command again to upload
+ # the index files. Always using the same target with rsync allows for
+ # very strict settings on the receiving server, you can literally specify
+ # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
+ # (serverwebroot is guaranteed to have a trailing slash in common.py)
+ logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
if subprocess.call(rsyncargs +
['--exclude', indexxml, '--exclude', indexjar,
repo_section, serverwebroot]) != 0:
sys.exit(1)
- # use stricter checking on the indexes since they provide the signature
- rsyncargs += ['--checksum']
- sectionpath = serverwebroot + repo_section
- if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0:
- sys.exit(1)
- if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0:
+ if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
sys.exit(1)
+ # upload "current version" symlinks if requested
+ if config['make_current_version_link'] and repo_section == 'repo':
+ links_to_upload = []
+ for f in glob.glob('*.apk') \
+ + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
+ if os.path.islink(f):
+ links_to_upload.append(f)
+ if len(links_to_upload) > 0:
+ if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
+ sys.exit(1)
def _local_sync(fromdir, todir):
- rsyncargs = ['rsync', '--recursive', '--links', '--times',
+ rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
'--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
# use stricter rsync checking on all files since people using offline mode
# are already prioritizing security above ease and speed
- rsyncargs += ['--checksum']
+ if not options.no_checksum:
+ rsyncargs.append('--checksum')
if options.verbose:
rsyncargs += ['--verbose']
if options.quiet:
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
+ parser.add_option("--no-checksum", action="store_true", default=False,
+ help="Don't use rsync checksums")
(options, args) = parser.parse_args()
config = common.read_config(options)
standardwebroot = True
for serverwebroot in config.get('serverwebroot', []):
- host, fdroiddir = serverwebroot.rstrip('/').split(':')
+ # this supports both an ssh host:path and just a path
+ s = serverwebroot.rstrip('/').split(':')
+ if len(s) == 1:
+ fdroiddir = s[0]
+ elif len(s) == 2:
+ host, fdroiddir = s
+ else:
+ logging.error('Malformed serverwebroot line: ' + serverwebroot)
+ sys.exit(1)
repobase = os.path.basename(fdroiddir)
if standardwebroot and repobase != 'fdroid':
logging.error('serverwebroot path does not end with "fdroid", '
--- /dev/null
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+#
+# gpgsign.py - part of the FDroid server tools
+# Copyright (C) 2015, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+from optparse import OptionParser
+import logging
+
+import common
+from common import FDroidPopen
+
+config = None
+options = None
+
+
+def main():
+
+ global config, options
+
+ # Parse command line...
+ parser = OptionParser(usage="Usage: %prog [options]")
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_option("-q", "--quiet", action="store_true", default=False,
+ help="Restrict output to warnings and errors")
+ (options, args) = parser.parse_args()
+
+ config = common.read_config(options)
+
+ repodirs = ['repo']
+ if config['archive_older'] != 0:
+ repodirs.append('archive')
+
+ signed = 0
+ for output_dir in repodirs:
+ if not os.path.isdir(output_dir):
+ logging.error("Missing output directory '" + output_dir + "'")
+ sys.exit(1)
+
+ unsigned = os.path.join(output_dir, 'index_unsigned.jar')
+ if os.path.exists(unsigned):
+
+ args = ['jarsigner', '-keystore', config['keystore'],
+ '-storepass:file', config['keystorepassfile'],
+ '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
+ unsigned, config['repo_keyalias']]
+ if config['keystore'] == 'NONE':
+ args += config['smartcardoptions']
+ else: # smardcards never use -keypass
+ args += ['-keypass:file', config['keypassfile']]
+ p = FDroidPopen(args)
+ if p.returncode != 0:
+ logging.critical("Failed to sign index")
+ sys.exit(1)
+ os.rename(unsigned, os.path.join(output_dir, 'index.jar'))
+ logging.info('Signed index in ' + output_dir)
+ signed += 1
+
+ if signed == 0:
+ logging.info("Nothing to do")
+
+if __name__ == "__main__":
+ main()
sys.exit(1)
# Get all metadata-defined apps...
- metaapps = [a for a in metadata.read_metadata().itervalues() if not a['Disabled']]
+ allmetaapps = [a for a in metadata.read_metadata().itervalues()]
+ metaapps = [a for a in allmetaapps if not a['Disabled']]
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
'apps': Counter(),
'appsver': Counter(),
'unknown': []
- }
+ }
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
rtype = common.getsrclibvcs(app['Repo'])
repotypes[rtype] += 1
f = open('stats/repotypes.txt', 'w')
- for rtype in repotypes:
- count = repotypes[rtype]
+ for rtype, count in repotypes.most_common():
f.write(rtype + ' ' + str(count) + '\n')
f.close()
checkmode = checkmode[:4]
ucms[checkmode] += 1
f = open('stats/update_check_modes.txt', 'w')
- for checkmode in ucms:
- count = ucms[checkmode]
+ for checkmode, count in ucms.most_common():
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
for category in app['Categories']:
ctgs[category] += 1
f = open('stats/categories.txt', 'w')
- for category in ctgs:
- count = ctgs[category]
+ for category, count in ctgs.most_common():
f.write(category + ' ' + str(count) + '\n')
f.close()
for antifeature in antifeatures:
afs[antifeature] += 1
f = open('stats/antifeatures.txt', 'w')
- for antifeature in afs:
- count = afs[antifeature]
+ for antifeature, count in afs.most_common():
f.write(antifeature + ' ' + str(count) + '\n')
f.close()
license = app['License']
licenses[license] += 1
f = open('stats/licenses.txt', 'w')
- for license in licenses:
- count = licenses[license]
+ for license, count in licenses.most_common():
f.write(license + ' ' + str(count) + '\n')
f.close()
+ # Write list of disabled apps...
+ logging.info("Processing disabled apps...")
+ disabled = [a['id'] for a in allmetaapps if a['Disabled']]
+ f = open('stats/disabled_apps.txt', 'w')
+ for appid in sorted(disabled):
+ f.write(appid + '\n')
+ f.close()
+
# Write list of latest apps added to the repo...
logging.info("Processing latest apps...")
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
- for app in latest:
- f.write(app + '\n')
+ for appid in latest:
+ f.write(appid + '\n')
f.close()
if unknownapks:
# -*- coding: utf-8 -*-
#
# update.py - part of the FDroid server tools
-# Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
import shutil
import glob
import re
+import socket
import zipfile
import hashlib
import pickle
+from datetime import datetime, timedelta
from xml.dom.minidom import Document
from optparse import OptionParser
import time
+from pyasn1.error import PyAsn1Error
+from pyasn1.codec.der import decoder, encoder
+from pyasn1_modules import rfc2315
+from hashlib import md5
+from binascii import hexlify, unhexlify
+
from PIL import Image
import logging
import common
import metadata
-from common import FDroidPopen, SilentPopen
+from common import FDroidPopen, SdkToolsPopen
from metadata import MetaDataException
if app['AntiFeatures']:
for af in app['AntiFeatures'].split(','):
wikidata += '{{AntiFeature|' + af + '}}\n'
- wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
+ wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
appid,
app['Name'],
time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
app['Source Code'],
app['Issue Tracker'],
app['Web Site'],
+ app['Changelog'],
app['Donate'],
app['FlattrID'],
app['Bitcoin'],
newpage = site.Pages[pagename]
newpage.save(text, summary='Auto-created')
except:
- logging.error("...FAILED to create page")
+ logging.error("...FAILED to create page '{0}'".format(pagename))
# Purge server cache to ensure counts are up to date
site.pages['Repository Maintenance'].purge()
resize_icon(iconpath, density)
+# A signature block file with a .DSA, .RSA, or .EC extension
+cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
+
+
+def getsig(apkpath):
+ """ Get the signing certificate of an apk. To get the same md5 has that
+ Android gets, we encode the .RSA certificate in a specific format and pass
+ it hex-encoded to the md5 digest algorithm.
+
+ :param apkpath: path to the apk
+ :returns: A string containing the md5 of the signature of the apk or None
+ if an error occurred.
+ """
+
+ cert = None
+
+ # verify the jar signature is correct
+ args = ['jarsigner', '-verify', apkpath]
+ p = FDroidPopen(args)
+ if p.returncode != 0:
+ logging.critical(apkpath + " has a bad signature!")
+ return None
+
+ with zipfile.ZipFile(apkpath, 'r') as apk:
+
+ certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
+
+ if len(certs) < 1:
+ logging.error("Found no signing certificates on %s" % apkpath)
+ return None
+ if len(certs) > 1:
+ logging.error("Found multiple signing certificates on %s" % apkpath)
+ return None
+
+ cert = apk.read(certs[0])
+
+ content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
+ if content.getComponentByName('contentType') != rfc2315.signedData:
+ logging.error("Unexpected format.")
+ return None
+
+ content = decoder.decode(content.getComponentByName('content'),
+ asn1Spec=rfc2315.SignedData())[0]
+ try:
+ certificates = content.getComponentByName('certificates')
+ except PyAsn1Error:
+ logging.error("Certificates not found.")
+ return None
+
+ cert_encoded = encoder.encode(certificates)[4:]
+
+ return md5(cert_encoded.encode('hex')).hexdigest()
+
+
def scan_apks(apps, apkcache, repodir, knownapks):
"""Scan the apks in the given repo directory.
logging.critical("Spaces in filenames are not allowed.")
sys.exit(1)
+ # Calculate the sha256...
+ sha = hashlib.sha256()
+ with open(apkfile, 'rb') as f:
+ while True:
+ t = f.read(16384)
+ if len(t) == 0:
+ break
+ sha.update(t)
+ shasum = sha.hexdigest()
+
+ usecache = False
if apkfilename in apkcache:
- logging.debug("Reading " + apkfilename + " from cache")
thisinfo = apkcache[apkfilename]
+ if thisinfo['sha256'] == shasum:
+ logging.debug("Reading " + apkfilename + " from cache")
+ usecache = True
+ else:
+ logging.debug("Ignoring stale cache data for " + apkfilename)
- else:
+ if not usecache:
logging.debug("Processing " + apkfilename)
thisinfo = {}
thisinfo['apkname'] = apkfilename
+ thisinfo['sha256'] = shasum
srcfilename = apkfilename[:-4] + "_src.tar.gz"
if os.path.exists(os.path.join(repodir, srcfilename)):
thisinfo['srcname'] = srcfilename
thisinfo['features'] = set()
thisinfo['icons_src'] = {}
thisinfo['icons'] = {}
- p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
+ p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
if p.returncode != 0:
if options.delete_unknown:
if os.path.exists(apkfile):
if common.isApkDebuggable(apkfile, config):
logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
- # Calculate the sha256...
- sha = hashlib.sha256()
- with open(apkfile, 'rb') as f:
- while True:
- t = f.read(1024)
- if len(t) == 0:
- break
- sha.update(t)
- thisinfo['sha256'] = sha.hexdigest()
-
# Get the signature (or md5 of, to be precise)...
- getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
- if not os.path.exists(getsig_dir + "/getsig.class"):
- logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
- sys.exit(1)
- p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
- 'getsig', os.path.join(os.getcwd(), apkfile)])
- thisinfo['sig'] = None
- for line in p.output.splitlines():
- if line.startswith('Result:'):
- thisinfo['sig'] = line[7:].strip()
- break
- if p.returncode != 0 or not thisinfo['sig']:
+ logging.debug('Getting signature of {0}'.format(apkfile))
+ thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
+ if not thisinfo['sig']:
logging.critical("Failed to get apk signature")
sys.exit(1)
apk = zipfile.ZipFile(apkfile, 'r')
+ # if an APK has files newer than the system time, suggest updating
+ # the system clock. This is useful for offline systems, used for
+ # signing, which do not have another source of clock sync info. It
+ # has to be more than 24 hours newer because ZIP/APK files do not
+ # store timezone info
+ info = apk.getinfo('AndroidManifest.xml')
+ dt_obj = datetime(*info.date_time)
+ checkdt = dt_obj - timedelta(1)
+ if datetime.today() < checkdt:
+ logging.warn('System clock is older than manifest in: '
+ + apkfilename + '\nSet clock to that time using:\n'
+ + 'sudo date -s "' + str(dt_obj) + '"')
+
iconfilename = "%s.%s.png" % (
thisinfo['id'],
thisinfo['versioncode'])
repo_pubkey_fingerprint = None
+# Generate a certificate fingerprint the same way keytool does it
+# (but with slightly different formatting)
+def cert_fingerprint(data):
+ digest = hashlib.sha256(data).digest()
+ ret = []
+ ret.append(' '.join("%02X" % ord(b) for b in digest))
+ return " ".join(ret)
+
+
+def extract_pubkey():
+ global repo_pubkey_fingerprint
+ if 'repo_pubkey' in config:
+ pubkey = unhexlify(config['repo_pubkey'])
+ else:
+ p = FDroidPopen(['keytool', '-exportcert',
+ '-alias', config['repo_keyalias'],
+ '-keystore', config['keystore'],
+ '-storepass:file', config['keystorepassfile']]
+ + config['smartcardoptions'], output=False)
+ if p.returncode != 0 or len(p.output) < 20:
+ msg = "Failed to get repo pubkey!"
+ if config['keystore'] == 'NONE':
+ msg += ' Is your crypto smartcard plugged in?'
+ logging.critical(msg)
+ sys.exit(1)
+ pubkey = p.output
+ repo_pubkey_fingerprint = cert_fingerprint(pubkey)
+ return hexlify(pubkey)
+
+
def make_index(apps, sortedids, apks, repodir, archive, categories):
"""Make a repo index.
el.appendChild(doc.createTextNode(value))
parent.appendChild(el)
+ def addElementNonEmpty(name, value, doc, parent):
+ if not value:
+ return
+ addElement(name, value, doc, parent)
+
def addElementCDATA(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createCDATASection(value))
repoel.setAttribute("url", config['repo_url'])
addElement('description', config['repo_description'], doc, repoel)
- repoel.setAttribute("version", "12")
+ repoel.setAttribute("version", "13")
repoel.setAttribute("timestamp", str(int(time.time())))
- if 'repo_keyalias' in config:
-
- # Generate a certificate fingerprint the same way keytool does it
- # (but with slightly different formatting)
- def cert_fingerprint(data):
- digest = hashlib.sha256(data).digest()
- ret = []
- ret.append(' '.join("%02X" % ord(b) for b in digest))
- return " ".join(ret)
-
- def extract_pubkey():
- p = FDroidPopen(['keytool', '-exportcert',
- '-alias', config['repo_keyalias'],
- '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile']]
- + config['smartcardoptions'], output=False)
- if p.returncode != 0:
- msg = "Failed to get repo pubkey!"
- if config['keystore'] == 'NONE':
- msg += ' Is your crypto smartcard plugged in?'
- logging.critical(msg)
- sys.exit(1)
- global repo_pubkey_fingerprint
- repo_pubkey_fingerprint = cert_fingerprint(p.output)
- return "".join("%02x" % ord(b) for b in p.output)
-
- repoel.setAttribute("pubkey", extract_pubkey())
+ nosigningkey = False
+ if not options.nosign:
+ if 'repo_keyalias' not in config:
+ nosigningkey = True
+ logging.critical("'repo_keyalias' not found in config.py!")
+ if 'keystore' not in config:
+ nosigningkey = True
+ logging.critical("'keystore' not found in config.py!")
+ if 'keystorepass' not in config and 'keystorepassfile' not in config:
+ nosigningkey = True
+ logging.critical("'keystorepass' not found in config.py!")
+ if 'keypass' not in config and 'keypassfile' not in config:
+ nosigningkey = True
+ logging.critical("'keypass' not found in config.py!")
+ if not os.path.exists(config['keystore']):
+ nosigningkey = True
+ logging.critical("'" + config['keystore'] + "' does not exist!")
+ if nosigningkey:
+ logging.warning("`fdroid update` requires a signing key, you can create one using:")
+ logging.warning("\tfdroid update --create-key")
+ sys.exit(1)
+ repoel.setAttribute("pubkey", extract_pubkey())
root.appendChild(repoel)
for appid in sortedids:
metadata.description_html(app['Description'], linkres),
doc, apel)
addElement('license', app['License'], doc, apel)
- if 'Categories' in app:
+ if 'Categories' in app and app['Categories']:
addElement('categories', ','.join(app["Categories"]), doc, apel)
# We put the first (primary) category in LAST, which will have
# the desired effect of making clients that only understand one
addElement('web', app['Web Site'], doc, apel)
addElement('source', app['Source Code'], doc, apel)
addElement('tracker', app['Issue Tracker'], doc, apel)
- if app['Donate']:
- addElement('donate', app['Donate'], doc, apel)
- if app['Bitcoin']:
- addElement('bitcoin', app['Bitcoin'], doc, apel)
- if app['Litecoin']:
- addElement('litecoin', app['Litecoin'], doc, apel)
- if app['Dogecoin']:
- addElement('dogecoin', app['Dogecoin'], doc, apel)
- if app['FlattrID']:
- addElement('flattr', app['FlattrID'], doc, apel)
+ addElementNonEmpty('changelog', app['Changelog'], doc, apel)
+ addElementNonEmpty('donate', app['Donate'], doc, apel)
+ addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
+ addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
+ addElementNonEmpty('dogecoin', app['Dogecoin'], doc, apel)
+ addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
# These elements actually refer to the current version (i.e. which
# one is recommended. They are historically mis-named, and need
if app['AntiFeatures']:
af = app['AntiFeatures'].split(',')
- # TODO: Temporarily not including UpstreamNonFree in the index,
- # because current F-Droid clients do not understand it, and also
- # look ugly when they encounter an unknown antifeature. This
- # filtering can be removed in time...
- if 'UpstreamNonFree' in af:
- af.remove('UpstreamNonFree')
if af:
- addElement('antifeatures', ','.join(af), doc, apel)
+ addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
if app['Provides']:
pv = app['Provides'].split(',')
- addElement('provides', ','.join(pv), doc, apel)
+ addElementNonEmpty('provides', ','.join(pv), doc, apel)
if app['Requires Root']:
addElement('requirements', 'root', doc, apel)
apklist[i]['apkname'], apklist[i + 1]['apkname']))
sys.exit(1)
+ current_version_code = 0
+ current_version_file = None
for apk in apklist:
+ # find the APK for the "Current Version"
+ if current_version_code < apk['versioncode']:
+ current_version_code = apk['versioncode']
+ if current_version_code < int(app['Current Version Code']):
+ current_version_file = apk['apkname']
+
apkel = doc.createElement("package")
apel.appendChild(apkel)
addElement('version', apk['version'], doc, apkel)
addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
if 'added' in apk:
addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
- if app['Requires Root']:
- if 'ACCESS_SUPERUSER' not in apk['permissions']:
- apk['permissions'].add('ACCESS_SUPERUSER')
-
- if len(apk['permissions']) > 0:
- addElement('permissions', ','.join(apk['permissions']), doc, apkel)
- if 'nativecode' in apk and len(apk['nativecode']) > 0:
+ addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
+ if 'nativecode' in apk:
addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
- if len(apk['features']) > 0:
- addElement('features', ','.join(apk['features']), doc, apkel)
+ addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
+
+ if current_version_file is not None \
+ and config['make_current_version_link'] \
+ and repodir == 'repo': # only create these
+ sanitized_name = re.sub('''[ '"&%?+=/]''', '',
+ app[config['current_version_name_source']])
+ apklinkname = sanitized_name + '.apk'
+ current_version_path = os.path.join(repodir, current_version_file)
+ if os.path.exists(apklinkname):
+ os.remove(apklinkname)
+ os.symlink(current_version_path, apklinkname)
+ # also symlink gpg signature, if it exists
+ for extension in ('.asc', '.sig'):
+ sigfile_path = current_version_path + extension
+ if os.path.exists(sigfile_path):
+ siglinkname = apklinkname + extension
+ if os.path.exists(siglinkname):
+ os.remove(siglinkname)
+ os.symlink(sigfile_path, siglinkname)
of = open(os.path.join(repodir, 'index.xml'), 'wb')
if options.pretty:
if 'repo_keyalias' in config:
- logging.info("Creating signed index with this key (SHA256):")
- logging.info("%s" % repo_pubkey_fingerprint)
+ if options.nosign:
+ logging.info("Creating unsigned index in preparation for signing")
+ else:
+ logging.info("Creating signed index with this key (SHA256):")
+ logging.info("%s" % repo_pubkey_fingerprint)
# Create a jar of the index...
- p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
+ jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
+ p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
if p.returncode != 0:
- logging.critical("Failed to create jar file")
+ logging.critical("Failed to create {0}".format(jar_output))
sys.exit(1)
# Sign the index...
- args = ['jarsigner', '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile'],
- '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
- os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
- if config['keystore'] == 'NONE':
- args += config['smartcardoptions']
- else: # smardcards never use -keypass
- args += ['-keypass:file', config['keypassfile']]
- p = FDroidPopen(args)
- # TODO keypass should be sent via stdin
- if p.returncode != 0:
- logging.critical("Failed to sign index")
- sys.exit(1)
+ signed = os.path.join(repodir, 'index.jar')
+ if options.nosign:
+ # Remove old signed index if not signing
+ if os.path.exists(signed):
+ os.remove(signed)
+ else:
+ args = ['jarsigner', '-keystore', config['keystore'],
+ '-storepass:file', config['keystorepassfile'],
+ '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
+ signed, config['repo_keyalias']]
+ if config['keystore'] == 'NONE':
+ args += config['smartcardoptions']
+ else: # smardcards never use -keypass
+ args += ['-keypass:file', config['keypassfile']]
+ p = FDroidPopen(args)
+ if p.returncode != 0:
+ logging.critical("Failed to sign index")
+ sys.exit(1)
# Copy the repo icon into the repo directory...
icon_dir = os.path.join(repodir, 'icons')
# Parse command line...
parser = OptionParser()
+ parser.add_option("--create-key", action="store_true", default=False,
+ help="Create a repo signing key in a keystore")
parser.add_option("-c", "--create-metadata", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("--delete-unknown", action="store_true", default=False,
help="Produce human-readable index.xml")
parser.add_option("--clean", action="store_true", default=False,
help="Clean update - don't uses caches, reprocess all apks")
+ parser.add_option("--nosign", action="store_true", default=False,
+ help="When configured for signed indexes, create only unsigned indexes at this stage")
(options, args) = parser.parse_args()
config = common.read_config(options)
logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
sys.exit(1)
+ # if the user asks to create a keystore, do it now, reusing whatever it can
+ if options.create_key:
+ if os.path.exists(config['keystore']):
+ logging.critical("Cowardily refusing to overwrite existing signing key setup!")
+ logging.critical("\t'" + config['keystore'] + "'")
+ sys.exit(1)
+
+ if 'repo_keyalias' not in config:
+ config['repo_keyalias'] = socket.getfqdn()
+ common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
+ if 'keydname' not in config:
+ config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
+ common.write_to_config(config, 'keydname', config['keydname'])
+ if 'keystore' not in config:
+ config['keystore'] = common.default_config.keystore
+ common.write_to_config(config, 'keystore', config['keystore'])
+
+ password = common.genpassword()
+ if 'keystorepass' not in config:
+ config['keystorepass'] = password
+ common.write_to_config(config, 'keystorepass', config['keystorepass'])
+ if 'keypass' not in config:
+ config['keypass'] = password
+ common.write_to_config(config, 'keypass', config['keypass'])
+ common.genkeystore(config)
+
# Get all apps...
apps = metadata.read_metadata()
f.write("Web Site:\n")
f.write("Source Code:\n")
f.write("Issue Tracker:\n")
+ f.write("Changelog:\n")
f.write("Summary:" + apk['name'] + "\n")
f.write("Description:\n")
f.write(apk['name'] + "\n")
if bestver == 0:
if app['Name'] is None:
- app['Name'] = appid
+ app['Name'] = app['Auto Name'] or appid
app['icon'] = None
logging.warn("Application " + appid + " has no packages")
else:
import sys
import os
-import shutil
-import subprocess
import glob
from optparse import OptionParser
import logging
import common
-from common import FDroidPopen, FDroidException
+from common import FDroidException
options = None
config = None
os.remove(remoteapk)
url = 'https://f-droid.org/repo/' + apkfilename
logging.info("...retrieving " + url)
- p = FDroidPopen(['wget', url], cwd=tmp_dir)
- if p.returncode != 0:
- raise FDroidException("Failed to get " + apkfilename)
-
- thisdir = os.path.join(tmp_dir, 'this_apk')
- thatdir = os.path.join(tmp_dir, 'that_apk')
- for d in [thisdir, thatdir]:
- if os.path.exists(d):
- shutil.rmtree(d)
- os.mkdir(d)
-
- if subprocess.call(['jar', 'xf',
- os.path.join("..", "..", unsigned_dir, apkfilename)],
- cwd=thisdir) != 0:
- raise FDroidException("Failed to unpack local build of " + apkfilename)
- if subprocess.call(['jar', 'xf',
- os.path.join("..", "..", remoteapk)],
- cwd=thatdir) != 0:
- raise FDroidException("Failed to unpack remote build of " + apkfilename)
-
- p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir)
- lines = p.output.splitlines()
- if len(lines) != 1 or 'META-INF' not in lines[0]:
- raise FDroidException("Unexpected diff output - " + p.output)
+ common.download_file(url, dldir=tmp_dir)
+
+ compare_result = common.compare_apks(
+ os.path.join(unsigned_dir, apkfilename),
+ remoteapk,
+ tmp_dir)
+ if compare_result:
+ raise FDroidException(compare_result)
logging.info("...successfully verified")
verified += 1
#!/bin/sh
#
-# Simple pre-commit hook to check that there are no errors in the fdroid
-# metadata files.
+# Simple pre-commit hook to check that there are no errors in the fdroidserver
+# source files.
# Redirect output to stderr.
exec 1>&2
-FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py"
+PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py"
+SH_FILES="hooks/pre-commit"
+BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion"
+RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
+
+# In the default configuration, the checks E123, E133, E226, E241 and E242 are
+# ignored because they are not rules unanimously accepted
+# On top of those, we ignore:
+# * E501: line too long (82 > 79 characters)
+# - Recommended for readability but not enforced
+# - Some lines are awkward to wrap around a char limit
+# * W503: line break before binary operator
+# - It's quite new
+# - Quite pedantic
+
+PEP8_IGNORE="E123,E133,E226,E241,E242,E501,W503"
+
+err() {
+ echo ERROR: "$@"
+ exit 1
+}
cmd_exists() {
command -v $1 1>/dev/null
}
-# For systems that switched to python3, first check for the python2 versions
if cmd_exists pyflakes-python2; then
PYFLAKES=pyflakes-python2
elif cmd_exists pyflakes; then
PYFLAKES=pyflakes
else
- echo "pyflakes is not installed!"
+ err "pyflakes is not installed!"
fi
if cmd_exists pep8-python2; then
elif cmd_exists pep8; then
PEP8=pep8
else
- echo "pep8 is not installed!"
+ err "pep8 is not installed!"
+fi
+
+if ! $PYFLAKES $PY_FILES; then
+ err "pyflakes tests failed!"
fi
-# If there are python errors or warnings, print them and fail.
-[ -n $PYFLAKES ] && $PYFLAKES $FILES
-[ -n $PEP8 ] && $PEP8 --ignore=E123,E501 $FILES
+if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then
+ err "pep8 tests failed!"
+fi
+
+
+for f in $SH_FILES; do
+ if ! dash -n $f; then
+ err "dash tests failed!"
+ fi
+done
+
+for f in $BASH_FILES; do
+ if ! bash -n $f; then
+ err "bash tests failed!"
+ fi
+done
+
+for f in $RB_FILES; do
+ if ! ruby -c $f 1>/dev/null; then
+ err "ruby tests failed!"
+ fi
+done
+
+exit 0
export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
+
#------------------------------------------------------------------------------#
-# run local build
-cd $WORKSPACE/fdroidserver/getsig
-./make.sh
+# run local tests, don't scan fdroidserver/ project for APKs
+# this is a local repo on the Guardian Project Jenkins server
+apksource=/var/www/fdroid
-#------------------------------------------------------------------------------#
-# run local tests
cd $WORKSPACE/tests
-./run-tests ~jenkins/
+./run-tests $apksource
#------------------------------------------------------------------------------#
-# test building the source tarball
+# test building the source tarball, then installing it
cd $WORKSPACE
python2 setup.py sdist
+rm -rf $WORKSPACE/env
+virtualenv --python=python2 $WORKSPACE/env
+. $WORKSPACE/env/bin/activate
+pip install dist/fdroidserver-*.tar.gz
+
+# run tests in new pip+virtualenv install
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
+
#------------------------------------------------------------------------------#
-# test install using site packages
+# test install using install direct from git repo
cd $WORKSPACE
rm -rf $WORKSPACE/env
virtualenv --python=python2 --system-site-packages $WORKSPACE/env
python2 setup.py install
# run tests in new pip+virtualenv install
-. $WORKSPACE/env/bin/activate
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
#------------------------------------------------------------------------------#
os.mkdir(cachedir)
cachefiles = [
- ('android-sdk_r23.0.2-linux.tgz',
- 'https://dl.google.com/android/android-sdk_r23.0.2-linux.tgz',
- 'a86741fee9140c340b60fe545566db7c0a43a0963f3c7e64d07b4d05ebbe89f4'),
+ ('android-sdk_r24.3.4-linux.tgz',
+ 'https://dl.google.com/android/android-sdk_r24.3.4-linux.tgz',
+ '886412375d8fe6e49a1583e57a8a36a47943666da681701ba9ad1ab7236e83ea'),
('gradle-1.4-bin.zip',
'https://services.gradle.org/distributions/gradle-1.4-bin.zip',
'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
('gradle-1.12-bin.zip',
'https://services.gradle.org/distributions/gradle-1.12-bin.zip',
'8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
+ ('gradle-2.1-bin.zip',
+ 'https://services.gradle.org/distributions/gradle-2.1-bin.zip',
+ '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
+ ('gradle-2.2.1-bin.zip',
+ 'https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
+ '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
+ ('gradle-2.3-bin.zip',
+ 'https://services.gradle.org/distributions/gradle-2.3-bin.zip',
+ '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
+ ('gradle-2.4-bin.zip',
+ 'https://services.gradle.org/distributions/gradle-2.4-bin.zip',
+ 'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
+ ('gradle-2.5-bin.zip',
+ 'https://services.gradle.org/distributions/gradle-2.5-bin.zip',
+ '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
+ ('gradle-2.6-bin.zip',
+ 'https://services.gradle.org/distributions/gradle-2.6-bin.zip',
+ '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
('Kivy-1.7.2.tar.gz',
'https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz',
- '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1')
- ]
+ '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'),
+]
if config['arch64']:
cachefiles.extend([
+ ('android-ndk-r10e-linux-x86_64.bin',
+ 'https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
+ '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
('android-ndk-r9b-linux-x86_64.tar.bz2',
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
'8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')])
else:
cachefiles.extend([
+ ('android-ndk-r10e-linux-x86.bin',
+ 'https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin',
+ '92b07d25aaad9b341a7f2b2a62402d508e948bf2dea3ee7b65a6aeb18bca7df5'),
('android-ndk-r9b-linux-x86.tar.bz2',
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2',
'748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'),
# Generate an appropriate Vagrantfile for the buildserver, based on our
# settings...
vagrantfile = """
-Vagrant::Config.run do |config|
+Vagrant.configure("2") do |config|
if Vagrant.has_plugin?("vagrant-cachier")
config.cache.scope = :box
config.vm.box = "{0}"
config.vm.box_url = "{1}"
- config.vm.customize ["modifyvm", :id, "--memory", "{2}"]
+ config.vm.provider "virtualbox" do |v|
+ v.customize ["modifyvm", :id, "--memory", "{2}"]
+ v.customize ["modifyvm", :id, "--cpus", "{3}"]
+ end
config.vm.provision :shell, :path => "fixpaths.sh"
-""".format(config['basebox'], config['baseboxurl'], config['memory'])
+""".format(config['basebox'],
+ config['baseboxurl'],
+ config['memory'],
+ config.get('cpus', 1))
if 'aptproxy' in config and config['aptproxy']:
vagrantfile += """
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
--- /dev/null
+[metadata]
+description-file = README.md
#!/usr/bin/env python2
from setuptools import setup
-import os
-import subprocess
import sys
-if not os.path.exists('fdroidserver/getsig/getsig.class'):
- subprocess.check_output('cd fdroidserver/getsig && javac getsig.java',
- shell=True)
+# workaround issue on OSX, where sys.prefix is not an installable location
+if sys.platform == 'darwin' and sys.prefix.startswith('/System'):
+ data_prefix = '.'
+else:
+ data_prefix = sys.prefix
setup(name='fdroidserver',
- version='0.2.1',
+ version='0.4.0',
description='F-Droid Server Tools',
- long_description=open('README').read(),
+ long_description=open('README.md').read(),
author='The F-Droid Project',
author_email='team@f-droid.org',
url='https://f-droid.org',
packages=['fdroidserver'],
scripts=['fdroid', 'fd-commit'],
data_files=[
- (sys.prefix + '/share/doc/fdroidserver/examples',
+ (data_prefix + '/share/doc/fdroidserver/examples',
['buildserver/config.buildserver.py',
'examples/config.py',
'examples/makebs.config.py',
'examples/opensc-fdroid.cfg',
'examples/fdroid-icon.png']),
- ('fdroidserver/getsig',
- ['fdroidserver/getsig/getsig.class']),
- ],
- install_requires=[
+ ],
+ install_requires=[ # should include 'python-magic' but its not strictly required
'mwclient',
'paramiko',
'Pillow',
- 'python-magic',
'apache-libcloud >= 0.14.1',
- ],
+ 'pyasn1',
+ 'pyasn1-modules',
+ 'requests',
+ ],
classifiers=[
'Development Status :: 3 - Alpha',
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
'Operating System :: POSIX',
'Topic :: Utilities',
- ],
+ ],
)
--- /dev/null
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import re
+import shutil
+import sys
+import tempfile
+import unittest
+
+localmodule = os.path.realpath(
+ os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+ sys.path.insert(0, localmodule)
+
+import fdroidserver.build
+import fdroidserver.common
+
+
+class BuildTest(unittest.TestCase):
+ '''fdroidserver/build.py'''
+
+ def _set_build_tools(self):
+ build_tools = os.path.join(fdroidserver.common.config['sdk_path'], 'build-tools')
+ if os.path.exists(build_tools):
+ fdroidserver.common.config['build_tools'] = ''
+ for f in sorted(os.listdir(build_tools), reverse=True):
+ versioned = os.path.join(build_tools, f)
+ if os.path.isdir(versioned) \
+ and os.path.isfile(os.path.join(versioned, 'aapt')):
+ fdroidserver.common.config['build_tools'] = versioned
+ break
+ return True
+ else:
+ print 'no build-tools found: ' + build_tools
+ return False
+
+ def _find_all(self):
+ for cmd in ('aapt', 'adb', 'android', 'zipalign'):
+ path = fdroidserver.common.find_sdk_tools_cmd(cmd)
+ if path is not None:
+ self.assertTrue(os.path.exists(path))
+ self.assertTrue(os.path.isfile(path))
+
+ def test_adapt_gradle(self):
+ testsbase = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
+ if not os.path.exists(testsbase):
+ os.makedirs(testsbase)
+ testsdir = tempfile.mkdtemp(prefix='test_adapt_gradle', dir=testsbase)
+ shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'),
+ os.path.join(testsdir, 'source-files'))
+ teststring = 'FAKE_VERSION_FOR_TESTING'
+ fdroidserver.build.config = {}
+ fdroidserver.build.config['build_tools'] = teststring
+ fdroidserver.build.adapt_gradle(testsdir)
+ pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring)
+ for f in ('source-files/fdroid/fdroidclient/build.gradle',
+ 'source-files/Zillode/syncthing-silk/build.gradle',
+ 'source-files/open-keychain/open-keychain/build.gradle',
+ 'source-files/osmandapp/osmand/build.gradle'):
+ filedata = open(os.path.join(testsdir, f)).read()
+ self.assertIsNotNone(pattern.search(filedata))
+ tp = os.path.join(testsdir,
+ 'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle')
+ filedata = open(tp).read()
+ self.assertIsNone(pattern.search(filedata))
+
+if __name__ == "__main__":
+ parser = optparse.OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+ newSuite = unittest.TestSuite()
+ newSuite.addTest(unittest.makeSuite(BuildTest))
+ unittest.main()
--- /dev/null
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import re
+import shutil
+import sys
+import tempfile
+import unittest
+
+localmodule = os.path.realpath(os.path.join(
+ os.path.dirname(inspect.getfile(inspect.currentframe())),
+ '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+ sys.path.insert(0,localmodule)
+
+import fdroidserver.common
+import fdroidserver.metadata
+
+class CommonTest(unittest.TestCase):
+ '''fdroidserver/common.py'''
+
+ def _set_build_tools(self):
+ build_tools = os.path.join(fdroidserver.common.config['sdk_path'], 'build-tools')
+ if os.path.exists(build_tools):
+ fdroidserver.common.config['build_tools'] = ''
+ for f in sorted(os.listdir(build_tools), reverse=True):
+ versioned = os.path.join(build_tools, f)
+ if os.path.isdir(versioned) \
+ and os.path.isfile(os.path.join(versioned, 'aapt')):
+ fdroidserver.common.config['build_tools'] = versioned
+ break
+ return True
+ else:
+ print 'no build-tools found: ' + build_tools
+ return False
+
+ def _find_all(self):
+ for cmd in ('aapt', 'adb', 'android', 'zipalign'):
+ path = fdroidserver.common.find_sdk_tools_cmd(cmd)
+ if path is not None:
+ self.assertTrue(os.path.exists(path))
+ self.assertTrue(os.path.isfile(path))
+
+ def test_find_sdk_tools_cmd(self):
+ fdroidserver.common.config = dict()
+ # TODO add this once everything works without sdk_path set in config
+ #self._find_all()
+ sdk_path = os.getenv('ANDROID_HOME')
+ if os.path.exists(sdk_path):
+ fdroidserver.common.config['sdk_path'] = sdk_path
+ if os.path.exists('/usr/bin/aapt'):
+ # this test only works when /usr/bin/aapt is installed
+ self._find_all()
+ build_tools = os.path.join(sdk_path, 'build-tools')
+ if self._set_build_tools():
+ self._find_all()
+ else:
+ print 'no build-tools found: ' + build_tools
+
+ def testIsApkDebuggable(self):
+ config = dict()
+ config['sdk_path'] = os.getenv('ANDROID_HOME')
+ fdroidserver.common.config = config
+ self._set_build_tools();
+ config['aapt'] = fdroidserver.common.find_sdk_tools_cmd('aapt')
+ # these are set debuggable
+ testfiles = []
+ testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip.apk'))
+ testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk'))
+ testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk'))
+ for apkfile in testfiles:
+ debuggable = fdroidserver.common.isApkDebuggable(apkfile, config)
+ self.assertTrue(debuggable,
+ "debuggable APK state was not properly parsed!")
+ # these are set NOT debuggable
+ testfiles = []
+ testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-release.apk'))
+ testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-release-unsigned.apk'))
+ for apkfile in testfiles:
+ debuggable = fdroidserver.common.isApkDebuggable(apkfile, config)
+ self.assertFalse(debuggable,
+ "debuggable APK state was not properly parsed!")
+
+ def testPackageNameValidity(self):
+ for name in ["org.fdroid.fdroid",
+ "org.f_droid.fdr0ID"]:
+ self.assertTrue(fdroidserver.common.is_valid_package_name(name),
+ "{0} should be a valid package name".format(name))
+ for name in ["0rg.fdroid.fdroid",
+ ".f_droid.fdr0ID",
+ "org.fdroid/fdroid",
+ "/org.fdroid.fdroid"]:
+ self.assertFalse(fdroidserver.common.is_valid_package_name(name),
+ "{0} should not be a valid package name".format(name))
+
+ def test_prepare_sources(self):
+ testint = 99999999
+ teststr = 'FAKE_STR_FOR_TESTING'
+
+ tmpdir = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
+ if not os.path.exists(tmpdir):
+ os.makedirs(tmpdir)
+ tmptestsdir = tempfile.mkdtemp(prefix='test_prepare_sources', dir=tmpdir)
+ shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'),
+ os.path.join(tmptestsdir, 'source-files'))
+
+ testdir = os.path.join(tmptestsdir, 'source-files', 'fdroid', 'fdroidclient')
+
+ config = dict()
+ config['sdk_path'] = os.getenv('ANDROID_HOME')
+ config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
+ fdroidserver.common.config = config
+ app = dict()
+ app['id'] = 'org.fdroid.froid'
+ build = dict(fdroidserver.metadata.flag_defaults)
+ build['commit'] = 'master'
+ build['forceversion'] = True
+ build['forcevercode'] = True
+ build['gradle'] = ['yes']
+ build['ndk_path'] = os.getenv('ANDROID_NDK_HOME')
+ build['target'] = 'android-' + str(testint)
+ build['type'] = 'gradle'
+ build['version'] = teststr
+ build['vercode'] = testint
+
+ class FakeVcs():
+ # no need to change to the correct commit here
+ def gotorevision(self, rev, refresh=True):
+ pass
+
+ # no srclib info needed, but it could be added...
+ def getsrclib(self):
+ return None
+
+ fdroidserver.common.prepare_source(FakeVcs(), app, build, testdir, testdir, testdir)
+
+ filedata = open(os.path.join(testdir, 'build.gradle')).read()
+ self.assertIsNotNone(re.search("\s+compileSdkVersion %s\s+" % testint, filedata))
+
+ filedata = open(os.path.join(testdir, 'AndroidManifest.xml')).read()
+ self.assertIsNone(re.search('android:debuggable', filedata))
+ self.assertIsNotNone(re.search('android:versionName="%s"' % build['version'], filedata))
+ self.assertIsNotNone(re.search('android:versionCode="%s"' % build['vercode'], filedata))
+
+
+if __name__ == "__main__":
+ parser = optparse.OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+ newSuite = unittest.TestSuite()
+ newSuite.addTest(unittest.makeSuite(CommonTest))
+ unittest.main()
--- /dev/null
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import sys
+import unittest
+
+localmodule = os.path.realpath(os.path.join(
+ os.path.dirname(inspect.getfile(inspect.currentframe())),
+ '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+ sys.path.insert(0,localmodule)
+
+import fdroidserver.common
+import fdroidserver.install
+
+class InstallTest(unittest.TestCase):
+ '''fdroidserver/install.py'''
+
+ def test_devices(self):
+ config = dict()
+ config['sdk_path'] = os.getenv('ANDROID_HOME')
+ fdroidserver.common.config = config
+ config['adb'] = fdroidserver.common.find_sdk_tools_cmd('adb')
+ self.assertTrue(os.path.exists(config['adb']))
+ self.assertTrue(os.path.isfile(config['adb']))
+ devices = fdroidserver.install.devices()
+ self.assertIsInstance(devices, list, 'install.devices() did not return a list!')
+ for device in devices:
+ self.assertIsInstance(device, basestring)
+
+
+if __name__ == "__main__":
+ parser = optparse.OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ (fdroidserver.install.options, args) = parser.parse_args(['--verbose'])
+ fdroidserver.common.options = fdroidserver.install.options
+
+ newSuite = unittest.TestSuite()
+ newSuite.addTest(unittest.makeSuite(InstallTest))
+ unittest.main()
copy_apks_into_repo() {
set +x
- for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned`; do
+ for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned -e badsig -e badcert`; do
name=$(basename $(dirname `dirname $f`))
apk=`$aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
test $f -nt repo/$apk && rm -f repo/$apk # delete existing if $f is newer
create_test_dir() {
test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles
- mktemp --directory --tmpdir=$WORKSPACE/.testfiles
+ TMPDIR=$WORKSPACE/.testfiles mktemp -d
}
create_test_file() {
test -e $WORKSPACE/.testfiles || mkdir $WORKSPACE/.testfiles
- mktemp --tmpdir=$WORKSPACE/.testfiles
+ TMPDIR=$WORKSPACE/.testfiles mktemp
}
#------------------------------------------------------------------------------#
# "main"
-if [ $1 = "-h" ] || [ $1 = "--help" ]; then
+if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
set +x
echo "Usage: $0 '/path/to/folder/with/apks'"
exit 1
fi
-if [ -z $ANDROID_HOME ]; then
+if [ -z "$ANDROID_HOME" ]; then
echo "ANDROID_HOME must be set with the path to the Android SDK, i.e.: "
echo " export ANDROID_HOME=/opt/android-sdk"
exit 1
fi
-if [ -z $1 ]; then
+if [ -z "$1" ]; then
APKDIR=`pwd`
else
APKDIR=$1
./hooks/pre-commit
+#------------------------------------------------------------------------------#
+echo_header "test python getsig replacement"
+
+cd $WORKSPACE/tests/getsig
+./make.sh
+for testcase in $WORKSPACE/tests/*.TestCase; do
+ $testcase
+done
+
+
+#------------------------------------------------------------------------------#
+echo_header "build the TeX manual"
+
+cd $WORKSPACE/docs
+./gendocs.sh -o html --email admin@f-droid.org fdroid "F-Droid Server Manual"
+
+
#------------------------------------------------------------------------------#
echo_header "create a source tarball and use that to build a repo"
REPOROOT=`create_test_dir`
cd $REPOROOT
tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1`
-cd $REPOROOT/fdroidserver-*/fdroidserver/getsig
-./make.sh
cd $REPOROOT
./fdroidserver-*/fdroid init
copy_apks_into_repo $REPOROOT
cd $REPOROOT
$fdroid init
$fdroid update --create-metadata
+$fdroid readmeta
$fdroid server update --local-copy-dir=/tmp/fdroid
# now test the errors work
$fdroid init
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+$fdroid readmeta
+grep -F '<application id=' repo/index.xml > /dev/null
LOCALCOPYDIR=`create_test_dir`/fdroid
$fdroid server update --local-copy-dir=$LOCALCOPYDIR
#------------------------------------------------------------------------------#
echo_header "check that 'fdroid init' fails when build-tools cannot be found"
-REPOROOT=`create_test_dir`
-FAKE_ANDROID_HOME=`create_test_dir`
-create_fake_android_home $FAKE_ANDROID_HOME
-rm -f $FAKE_ANDROID_HOME/build-tools/*/aapt
-KEYSTORE=$REPOROOT/keystore.jks
-cd $REPOROOT
-set +e
-$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
-[ $? -eq 0 ] && exit 1
-set -e
+if [ -e /usr/bin/aapt ]; then
+ echo "/usr/bin/aapt exists, not running test"
+else
+ REPOROOT=`create_test_dir`
+ FAKE_ANDROID_HOME=`create_test_dir`
+ create_fake_android_home $FAKE_ANDROID_HOME
+ rm -f $FAKE_ANDROID_HOME/build-tools/*/aapt
+ KEYSTORE=$REPOROOT/keystore.jks
+ cd $REPOROOT
+ set +e
+ $fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME
+ [ $? -eq 0 ] && exit 1
+ set -e
+fi
#------------------------------------------------------------------------------#
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+$fdroid readmeta
+grep -F '<application id=' repo/index.xml > /dev/null
test -e repo/index.xml
test -e repo/index.jar
export ANDROID_HOME=$STORED_ANDROID_HOME
copy_apks_into_repo $REPOROOT
$fdroid init
$fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+$fdroid readmeta
+grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
+$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo manually and generate a keystore"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+touch config.py
+cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/
+! test -e $KEYSTORE
+set +e
+$fdroid update
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has no keystore!"
+ exit 1
+else
+ echo '`fdroid update` prompted to add keystore'
+fi
+set -e
+$fdroid update --create-key
+test -e $KEYSTORE
+copy_apks_into_repo $REPOROOT
+$fdroid update --create-metadata
+$fdroid readmeta
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
+$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
-cp $WORKSPACE/tests/urzip.apk $REPOROOT/
+grep -F '<application id=' repo/index.xml > /dev/null
+test -e $REPOROOT/repo/info.guardianproject.urzip_100.apk || \
+ cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
$fdroid update --create-metadata
+$fdroid readmeta
test -e repo/index.xml
test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
test ! -e NONE
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo with no keystore, add APK, and update"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+touch config.py
+touch fdroid-icon.png
+mkdir repo
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
+set +e
+$fdroid update --create-metadata
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has no keystore!"
+ exit 1
+else
+ echo '`fdroid update` prompted to add keystore'
+fi
+set -e
+
+# now set up fake, non-working keystore setup
+touch $KEYSTORE
+echo "keystore = \"$KEYSTORE\"" >> config.py
+echo 'repo_keyalias = "foo"' >> config.py
+echo 'keystorepass = "foo"' >> config.py
+echo 'keypass = "foo"' >> config.py
+set +e
+$fdroid update --create-metadata
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has a bad/fake keystore!"
+ exit 1
+else
+ echo '`fdroid update` prompted to add keystore'
+fi
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo with keystore with APK, update, then without key"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE
+test -e $KEYSTORE
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
+$fdroid update --create-metadata
+$fdroid readmeta
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+
+# now set fake repo_keyalias
+sed -i 's,^ *repo_keyalias.*,repo_keyalias = "fake",' $REPOROOT/config.py
+set +e
+$fdroid update
+if [ $? -eq 0 ]; then
+ echo "This should have failed because this repo has a bad repo_keyalias!"
+ exit 1
+else
+ echo '`fdroid update` prompted to add keystore'
+fi
+set -e
+
+# try creating a new keystore, but fail because the old one is there
+test -e $KEYSTORE
+set +e
+$fdroid update --create-key
+if [ $? -eq 0 ]; then
+ echo "This should have failed because a keystore is already there!"
+ exit 1
+else
+ echo '`fdroid update` complained about existing keystore'
+fi
+set -e
+
+# now actually create the key with the existing settings
+rm -f $KEYSTORE
+! test -e $KEYSTORE
+$fdroid update --create-key
+test -e $KEYSTORE
+
+
+#------------------------------------------------------------------------------#
+
+# remove this to prevent git conflicts and complaining
+rm -rf $WORKSPACE/fdroidserver.egg-info/
+
echo SUCCESS
--- /dev/null
+/*
+ * Copyright (c) 2015 OpenSilk Productions LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+ repositories {
+ mavenCentral()
+ jcenter()
+ }
+ dependencies {
+ classpath 'com.android.tools.build:gradle:1.1.3'
+ classpath 'me.tatarka:gradle-retrolambda:2.5.0'
+ classpath 'org.robolectric:robolectric-gradle-plugin:1.0.1'
+
+ // NOTE: Do not place your application dependencies here; they belong
+ // in the individual module build.gradle files
+ }
+}
+
+allprojects {
+ repositories {
+ mavenCentral()
+ jcenter()
+ maven { url '../../m2/repository' }
+ maven { url 'https://oss.sonatype.org/content/repositories/snapshots' }
+ }
+}
+
+// Build config
+ext.compileSdkVersion = 22
+ext.buildToolsVersion = "22.0.1"
+
+// defaultConfig
+ext.targetSdkVersion = 22
+
+ext.supportLibVersion = "22.1.1"
+ext.dagger2Version = "2.0"
+ext.rxAndroidVersion = "0.23.0"
+ext.timberVersion = "2.5.0"
+ext.commonsLangVersion = "3.3.2"
+ext.butterKnifeVersion = "6.0.0"
+ext.commonsIoVersion = "2.4"
+ext.gsonVersion = "2.3"
+
+def gitSha() {
+ return 'git rev-parse --short HEAD'.execute().text.trim()
+}
+
+def getDebugVersionSuffix() {
+ return "${gitSha()}".isEmpty() ? "-SNAPSHOT" : "-SNAPSHOT-${gitSha()}"
+}
--- /dev/null
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:tools="http://schemas.android.com/tools"
+ package="org.fdroid.fdroid"
+ android:installLocation="auto"
+ android:versionCode="940"
+ android:versionName="0.94-test"
+ >
+
+ <uses-sdk
+ tools:overrideLibrary="org.thoughtcrime.ssl.pinning"
+ android:minSdkVersion="8"
+ android:targetSdkVersion="21"
+ />
+
+ <supports-screens
+ android:anyDensity="true"
+ android:largeScreens="true"
+ android:normalScreens="true"
+ android:resizeable="true"
+ android:smallScreens="true"
+ android:xlargeScreens="true"
+ />
+
+ <uses-feature
+ android:name="android.hardware.telephony"
+ android:required="false" />
+ <uses-feature
+ android:name="android.hardware.wifi"
+ android:required="false" />
+ <uses-feature
+ android:name="android.hardware.touchscreen"
+ android:required="false" />
+ <uses-feature
+ android:name="android.hardware.nfc"
+ android:required="false" />
+ <uses-feature
+ android:name="android.hardware.bluetooth"
+ android:required="false" />
+
+ <uses-permission android:name="android.permission.INTERNET" />
+ <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+ <uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
+ <uses-permission android:name="android.permission.CHANGE_WIFI_MULTICAST_STATE" />
+ <uses-permission android:name="android.permission.CHANGE_WIFI_STATE" />
+ <uses-permission android:name="android.permission.BLUETOOTH" />
+ <uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
+ <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"
+ android:maxSdkVersion="18" />
+ <uses-permission android:name="android.permission.NFC" />
+
+ <!-- These permissions are only granted when F-Droid is installed as a system-app! -->
+ <uses-permission android:name="android.permission.INSTALL_PACKAGES"
+ tools:ignore="ProtectedPermissions"/>
+ <uses-permission android:name="android.permission.DELETE_PACKAGES"
+ tools:ignore="ProtectedPermissions"/>
+
+ <application
+ android:debuggable="true"
+ android:name="FDroidApp"
+ android:icon="@drawable/ic_launcher"
+ android:label="@string/app_name"
+ android:description="@string/app_description"
+ android:allowBackup="true"
+ android:theme="@style/AppThemeDark"
+ android:supportsRtl="true"
+ >
+
+ <provider
+ android:authorities="org.fdroid.fdroid.data.AppProvider"
+ android:name="org.fdroid.fdroid.data.AppProvider"
+ android:exported="false"/>
+
+ <provider
+ android:authorities="org.fdroid.fdroid.data.RepoProvider"
+ android:name="org.fdroid.fdroid.data.RepoProvider"
+ android:exported="false"/>
+
+ <provider
+ android:authorities="org.fdroid.fdroid.data.ApkProvider"
+ android:name="org.fdroid.fdroid.data.ApkProvider"
+ android:exported="false"/>
+
+ <provider
+ android:authorities="org.fdroid.fdroid.data.InstalledAppProvider"
+ android:name="org.fdroid.fdroid.data.InstalledAppProvider"
+ android:exported="false"/>
+
+ <activity
+ android:name=".FDroid"
+ android:launchMode="singleTop"
+ android:configChanges="keyboardHidden|orientation|screenSize" >
+
+ <!-- App URLs -->
+ <intent-filter>
+ <action android:name="android.intent.action.MAIN" />
+
+ <category android:name="android.intent.category.LAUNCHER" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="fdroid.app" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="http" />
+ <data android:scheme="https" />
+ <data android:host="f-droid.org" />
+ <data android:host="www.f-droid.org" />
+ <data android:pathPrefix="/app/" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="http" />
+ <data android:scheme="https" />
+ <data android:host="f-droid.org" />
+ <data android:host="www.f-droid.org" />
+ <data android:pathPrefix="/repository/browse" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="market" android:host="details" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="http" />
+ <data android:scheme="https" />
+ <data android:host="play.google.com" /> <!-- they don't do www. -->
+ <data android:path="/store/apps/details" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="amzn" android:host="apps" android:path="/android" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="http" />
+ <data android:scheme="https" />
+ <data android:host="amazon.com" />
+ <data android:host="www.amazon.com" />
+ <data android:path="/gp/mas/dl/android" />
+ </intent-filter>
+
+ <!-- Search URLs -->
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="fdroid.search" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="market" android:host="search" />
+ </intent-filter>
+
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+ <category android:name="android.intent.category.BROWSABLE" />
+
+ <data android:scheme="http" />
+ <data android:scheme="https" />
+ <data android:host="play.google.com" /> <!-- they don't do www. -->
+ <data android:path="/store/search" />
+ </intent-filter>
+
+ <!-- Handle NFC tags detected from outside our application -->
+ <intent-filter>
+ <action android:name="android.nfc.action.NDEF_DISCOVERED" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+
+ <!--
+ URIs that come in via NFC have scheme/host normalized to all lower case
+ https://developer.android.com/reference/android/nfc/NfcAdapter.html#ACTION_NDEF_DISCOVERED
+ -->
+ <data android:scheme="fdroidrepo" />
+ <data android:scheme="fdroidrepos" />
+ </intent-filter>
+
+ <!-- Repo URLs -->
+
+ <!--
+ This intent serves two purposes: Swapping apps between devices and adding a
+ repo from a website (e.g. https://guardianproject.info/fdroid/repo).
+ We intercept both of these situations in the FDroid activity, and then redirect
+ to the appropriate handler (swap handling, manage repos respectively) from there.
+
+ The reason for this is that the only differentiating factor is the presence
+ of a "swap=1" in the query string, and intent-filter is unable to deal with
+ query parameters. An alternative would be to do something like fdroidswap:// as
+ a scheme, but then we. Need to copy/paste all of this intent-filter stuff and
+ keep it up to date when it changes or a bug is found.
+ -->
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.BROWSABLE" />
+ <category android:name="android.intent.category.DEFAULT" />
+
+ <!--
+ Android's scheme matcher is case-sensitive, so include
+ ALL CAPS versions to support ALL CAPS URLs in QR Codes.
+ QR Codes have a special ALL CAPS mode that uses a reduced
+ character set, making for more compact QR Codes.
+ -->
+ <data android:scheme="http" />
+ <data android:scheme="HTTP" />
+ <data android:scheme="https" />
+ <data android:scheme="HTTPS" />
+ <data android:scheme="fdroidrepo" />
+ <data android:scheme="FDROIDREPO" />
+ <data android:scheme="fdroidrepos" />
+ <data android:scheme="FDROIDREPOS" />
+
+ <data android:host="*" />
+
+ <!--
+ The pattern matcher here is poorly implemented, in particular the * is
+ non-greedy, so you have to do stupid tricks to match patterns that have
+ repeat characters in them. http://stackoverflow.com/a/8599921/306864
+ -->
+ <data android:path="/fdroid/repo" />
+ <data android:pathPattern="/fdroid/repo/*" />
+ <data android:pathPattern="/.*/fdroid/repo" />
+ <data android:pathPattern="/.*/fdroid/repo/*" />
+ <data android:pathPattern="/.*/.*/fdroid/repo" />
+ <data android:pathPattern="/.*/.*/fdroid/repo/*" />
+ <data android:pathPattern="/.*/.*/.*/fdroid/repo" />
+ <data android:pathPattern="/.*/.*/.*/fdroid/repo/*" />
+ <data android:path="/fdroid/archive" />
+ <data android:pathPattern="/fdroid/archive/*" />
+ <data android:pathPattern="/.*/fdroid/archive" />
+ <data android:pathPattern="/.*/fdroid/archive/*" />
+ <data android:pathPattern="/.*/.*/fdroid/archive" />
+ <data android:pathPattern="/.*/.*/fdroid/archive/*" />
+ <data android:pathPattern="/.*/.*/.*/fdroid/archive" />
+ <data android:pathPattern="/.*/.*/.*/fdroid/archive/*" />
+ <!--
+ Some QR Code scanners don't respect custom schemes like fdroidrepo://,
+ so this is a workaround, since the local repo URL is all uppercase in
+ the QR Code for sending the local repo to another device.
+ -->
+ <data android:path="/FDROID/REPO" />
+ <data android:pathPattern="/.*/FDROID/REPO" />
+ <data android:pathPattern="/.*/.*/FDROID/REPO" />
+ <data android:pathPattern="/.*/.*/.*/FDROID/REPO" />
+ </intent-filter>
+
+ <meta-data
+ android:name="android.app.default_searchable"
+ android:value=".SearchResults" />
+ </activity>
+ <activity
+ android:name=".views.swap.ConnectSwapActivity"
+ android:theme="@style/SwapTheme.Wizard.ReceiveSwap"
+ android:label=""
+ android:noHistory="true"
+ android:parentActivityName=".FDroid"
+ android:screenOrientation="portrait"
+ android:configChanges="orientation|keyboardHidden">
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ </activity>
+ <activity
+ android:name=".installer.InstallConfirmActivity"
+ android:label="@string/menu_install"
+ android:parentActivityName=".FDroid">
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ </activity>
+ <activity
+ android:name=".views.ManageReposActivity"
+ android:label="@string/app_name"
+ android:launchMode="singleTask"
+ android:parentActivityName=".FDroid" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ <intent-filter>
+ <action android:name="android.intent.action.VIEW" />
+
+ <category android:name="android.intent.category.DEFAULT" />
+
+ <data android:mimeType="application/vnd.org.fdroid.fdroid.repo" />
+ </intent-filter>
+ </activity>
+ <activity
+ android:name=".NfcNotEnabledActivity"
+ android:noHistory="true" />
+ <!--<activity android:name=".views.QrWizardDownloadActivity" />
+ <activity android:name=".views.QrWizardWifiNetworkActivity" />
+ <activity
+ android:name=".views.LocalRepoActivity"
+ android:configChanges="orientation|keyboardHidden|screenSize"
+ android:label="@string/local_repo"
+ android:launchMode="singleTop"
+ android:parentActivityName=".FDroid"
+ android:screenOrientation="portrait" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ </activity>
+ <activity
+ android:name=".views.SelectLocalAppsActivity"
+ android:label="@string/setup_repo"
+ android:parentActivityName=".views.LocalRepoActivity" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".views.LocalRepoActivity" />
+ </activity>-->
+ <activity
+ android:name=".views.RepoDetailsActivity"
+ android:label="@string/menu_manage"
+ android:parentActivityName=".views.ManageReposActivity"
+ android:windowSoftInputMode="stateHidden">
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".views.ManageReposActivity" />
+ </activity>
+
+ <activity
+ android:name=".AppDetails"
+ android:label="@string/app_details"
+ android:exported="true"
+ android:parentActivityName=".FDroid" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+
+ </activity>
+ <activity
+ android:name=".views.swap.SwapAppListActivity$SwapAppDetails"
+ android:label="@string/app_details"
+ android:parentActivityName=".views.swap.SwapAppListActivity" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".views.swap.SwapAppListActivity" />
+ </activity>
+ <activity
+ android:label="@string/menu_preferences"
+ android:name=".PreferencesActivity"
+ android:parentActivityName=".FDroid" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ </activity>
+ <activity
+ android:label="@string/menu_swap"
+ android:name=".views.swap.SwapActivity"
+ android:parentActivityName=".FDroid"
+ android:theme="@style/SwapTheme.Wizard"
+ android:screenOrientation="portrait"
+ android:configChanges="orientation|keyboardHidden">
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ </activity>
+ <activity
+ android:label="@string/swap"
+ android:name=".views.swap.SwapAppListActivity"
+ android:parentActivityName=".FDroid"
+ android:theme="@style/SwapTheme.AppList"
+ android:screenOrientation="portrait"
+ android:configChanges="orientation|keyboardHidden">
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+ </activity>
+ <!-- Note: Theme.NoDisplay, this activity shows dialogs only -->
+ <activity
+ android:name=".installer.InstallIntoSystemDialogActivity"
+ android:theme="@android:style/Theme.NoDisplay" />
+ <receiver
+ android:name=".installer.InstallIntoSystemBootReceiver" >
+ <intent-filter>
+ <action android:name="android.intent.action.BOOT_COMPLETED" />
+ </intent-filter>
+ </receiver>
+ <activity
+ android:name=".SearchResults"
+ android:label="@string/search_results"
+ android:exported="true"
+ android:launchMode="singleTop"
+ android:parentActivityName=".FDroid" >
+ <meta-data
+ android:name="android.support.PARENT_ACTIVITY"
+ android:value=".FDroid" />
+
+ <intent-filter>
+ <action android:name="android.intent.action.SEARCH" />
+ </intent-filter>
+
+ <meta-data
+ android:name="android.app.searchable"
+ android:resource="@xml/searchable" />
+ </activity>
+
+ <receiver android:name=".receiver.StartupReceiver" >
+ <intent-filter>
+ <action android:name="android.intent.action.BOOT_COMPLETED" />
+
+ <category android:name="android.intent.category.HOME" />
+ </intent-filter>
+ </receiver>
+ <receiver android:name=".receiver.PackageAddedReceiver" >
+ <intent-filter>
+ <action android:name="android.intent.action.PACKAGE_ADDED" />
+
+ <data android:scheme="package" />
+ </intent-filter>
+ </receiver>
+ <receiver android:name=".receiver.PackageUpgradedReceiver" >
+ <intent-filter>
+ <action android:name="android.intent.action.PACKAGE_REPLACED" />
+
+ <data android:scheme="package" />
+ </intent-filter>
+ </receiver>
+ <receiver android:name=".receiver.PackageRemovedReceiver" >
+ <intent-filter>
+ <action android:name="android.intent.action.PACKAGE_REMOVED" />
+
+ <data android:scheme="package" />
+ </intent-filter>
+ </receiver>
+ <receiver android:name=".receiver.WifiStateChangeReceiver" >
+ <intent-filter>
+ <action android:name="android.net.wifi.STATE_CHANGE" />
+ </intent-filter>
+ </receiver>
+
+ <service android:name=".UpdateService" />
+ <service android:name=".net.WifiStateChangeService" />
+ <service android:name=".localrepo.LocalRepoService" />
+ </application>
+
+</manifest>
--- /dev/null
+apply plugin: 'com.android.application'
+
+if ( !hasProperty( 'sourceDeps' ) ) {
+
+ logger.info "Setting up *binary* dependencies for F-Droid (if you'd prefer to build from source, pass the -PsourceDeps argument to gradle while building)."
+
+ repositories {
+ jcenter()
+
+ // This is here until we sort out all dependencies from mavenCentral/jcenter. Once all of
+ // the dependencies below have been sorted out, this can be removed.
+ flatDir {
+ dirs 'libs/binaryDeps'
+ }
+ }
+
+ dependencies {
+
+ compile 'com.android.support:support-v4:22.1.0',
+ 'com.android.support:appcompat-v7:22.1.0',
+ 'com.android.support:support-annotations:22.1.0',
+
+ 'org.thoughtcrime.ssl.pinning:AndroidPinning:1.0.0',
+ 'com.nostra13.universalimageloader:universal-image-loader:1.9.4',
+ 'com.google.zxing:core:3.2.0',
+ 'eu.chainfire:libsuperuser:1.0.0.201504231659',
+
+ // We use a slightly modified spongycastle, see
+ // openkeychain/spongycastle with some changes on top of 1.51.0.0
+ 'com.madgag.spongycastle:pkix:1.51.0.0',
+ 'com.madgag.spongycastle:prov:1.51.0.0',
+ 'com.madgag.spongycastle:core:1.51.0.0'
+
+ // Upstream doesn't have a binary on mavenCentral/jcenter yet:
+ // https://github.com/kolavar/android-support-v4-preferencefragment/issues/13
+ compile(name: 'support-v4-preferencefragment-release', ext: 'aar')
+
+ // Fork for F-Droid, including support for https. Not merged into upstream
+ // yet (seems to be a little unsupported as of late), so not using mavenCentral/jcenter.
+ compile(name: 'nanohttpd-2.1.0')
+
+ // Upstream doesn't have a binary on mavenCentral.
+ compile(name: 'zipsigner')
+
+ // Project semi-abandoned, 3.4.1 is from 2011 and we use trunk from 2013
+ compile(name: 'jmdns')
+
+ androidTestCompile 'commons-io:commons-io:2.2'
+ }
+
+} else {
+
+ logger.info "Setting up *source* dependencies for F-Droid (because you passed in the -PsourceDeps argument to gradle while building)."
+
+ repositories {
+ jcenter()
+ }
+
+ dependencies {
+ compile project(':extern:AndroidPinning')
+ compile project(':extern:UniversalImageLoader:library')
+ compile project(':extern:libsuperuser:libsuperuser')
+ compile project(':extern:nanohttpd:core')
+ compile project(':extern:jmdns')
+ compile project(':extern:zipsigner')
+ compile project(':extern:zxing-core')
+ compile( project(':extern:support-v4-preferencefragment') ) {
+ exclude module: 'support-v4'
+ }
+
+ // Until the android team updates the gradle plugin version from 0.10.0 to
+ // a newer version, we can't build this from source with our gradle version
+ // of 1.0.0. They use API's which have been moved in the newer plugin.
+ // So yes, this is a little annoying that our "source dependencies" include
+ // a bunch of binaries from jcenter - but the ant build file (which is the
+ // one used to build F-Droid which is distributed on https://f-droid.org
+ // builds these from source - well - not support-v4).
+ //
+ // If the android team gets the build script working with the newer plugin,
+ // then you can find the relevant portions of the ../build.gradle file that
+ // include magic required to make it work at around about the v0.78 git tag.
+ // They have since been removed to clean up the build file.
+ compile 'com.android.support:support-v4:22.1.0',
+ 'com.android.support:appcompat-v7:22.1.0',
+ 'com.android.support:support-annotations:22.1.0'
+
+ androidTestCompile 'commons-io:commons-io:2.2'
+ }
+
+}
+
+task cleanBinaryDeps(type: Delete) {
+
+ enabled = project.hasProperty('sourceDeps')
+ description = "Removes all .jar and .aar files from F-Droid/libs/. Requires the sourceDeps property to be set (\"gradle -PsourceDeps cleanBinaryDeps\")"
+
+ delete fileTree('libs/binaryDeps') {
+ include '*.aar'
+ include '*.jar'
+ }
+}
+
+task binaryDeps(type: Copy, dependsOn: ':F-Droid:prepareReleaseDependencies') {
+
+ enabled = project.hasProperty('sourceDeps')
+ description = "Copies .jar and .aar files from subproject dependencies in extern/ to F-Droid/libs. Requires the sourceDeps property to be set (\"gradle -PsourceDeps binaryDeps\")"
+
+ from ('../extern/' ) {
+ include 'support-v4-preferencefragment/build/outputs/aar/support-v4-preferencefragment-release.aar',
+ 'nanohttpd/core/build/libs/nanohttpd-2.1.0.jar',
+ 'zipsigner/build/libs/zipsigner.jar',
+ 'jmdns/build/libs/jmdns.jar',
+ 'Support/v4/build/libs/support-v4.jar'
+ }
+
+ into 'libs/binaryDeps'
+
+ includeEmptyDirs false
+
+ eachFile { FileCopyDetails details ->
+ // Don't copy to a sub folder such as libs/binaryDeps/Project/build/outputs/aar/project.aar, but
+ // rather libs/binaryDeps/project.aar.
+ details.path = details.name
+ }
+
+}
+
+android {
+ compileSdkVersion 21
+ buildToolsVersion '22.0.1'
+
+ sourceSets {
+ main {
+ manifest.srcFile 'AndroidManifest.xml'
+ java.srcDirs = ['src']
+ resources.srcDirs = ['src']
+ aidl.srcDirs = ['src']
+ renderscript.srcDirs = ['src']
+ res.srcDirs = ['res']
+ assets.srcDirs = ['assets']
+ }
+
+ androidTest.setRoot('test')
+ androidTest {
+ manifest.srcFile 'test/AndroidManifest.xml'
+ java.srcDirs = ['test/src']
+ resources.srcDirs = ['test/src']
+ aidl.srcDirs = ['test/src']
+ renderscript.srcDirs = ['test/src']
+ res.srcDirs = ['test/res']
+ assets.srcDirs = ['test/assets']
+ }
+ }
+
+ buildTypes {
+ release {
+ minifyEnabled false
+ }
+ buildTypes {
+ debug {
+ debuggable true
+ }
+ }
+ }
+
+ compileOptions {
+ compileOptions.encoding = "UTF-8"
+
+ // Use Java 1.7, requires minSdk 8
+ sourceCompatibility JavaVersion.VERSION_1_7
+ targetCompatibility JavaVersion.VERSION_1_7
+ }
+
+ lintOptions {
+ checkReleaseBuilds false
+ abortOnError false
+ }
+
+ // Enable all Android lint warnings
+ gradle.projectsEvaluated {
+ tasks.withType(JavaCompile) {
+ options.compilerArgs << "-Xlint:all"
+ }
+ }
+
+}
+
+// This person took the example code below from another blogpost online, however
+// I lost the reference to it:
+// http://stackoverflow.com/questions/23297562/gradle-javadoc-and-android-documentation
+android.applicationVariants.all { variant ->
+
+ task("generate${variant.name}Javadoc", type: Javadoc) {
+ title = "$name $version API"
+ description "Generates Javadoc for F-Droid."
+ source = variant.javaCompile.source
+
+ def sdkDir
+ Properties properties = new Properties()
+ File localProps = project.rootProject.file('local.properties')
+ if (localProps.exists()) {
+ properties.load(localProps.newDataInputStream())
+ sdkDir = properties.getProperty('sdk.dir')
+ } else {
+ sdkDir = System.getenv('ANDROID_HOME')
+ }
+ if (!sdkDir) {
+ throw new ProjectConfigurationException("Cannot find android sdk. Make sure sdk.dir is defined in local.properties or the environment variable ANDROID_HOME is set.", null)
+ }
+
+ ext.androidJar = "${sdkDir}/platforms/${android.compileSdkVersion}/android.jar"
+ classpath = files(variant.javaCompile.classpath.files) + files(ext.androidJar)
+ options.links("http://docs.oracle.com/javase/7/docs/api/");
+ options.links("http://d.android.com/reference/");
+ exclude '**/BuildConfig.java'
+ exclude '**/R.java'
+ }
+}
--- /dev/null
+apply plugin: 'com.android.application'
+apply plugin: 'witness'
+apply plugin: 'jacoco'
+apply plugin: 'com.github.kt3k.coveralls'
+
+dependencies {
+ // NOTE: Always use fixed version codes not dynamic ones, e.g. 0.7.3 instead of 0.7.+, see README for more information
+ // NOTE: libraries are pinned to a specific build, see below
+
+ // from local Android SDK
+ compile 'com.android.support:support-v4:22.1.1'
+ compile 'com.android.support:appcompat-v7:22.1.1'
+ compile 'com.android.support:recyclerview-v7:22.1.0'
+ compile 'com.android.support:cardview-v7:22.1.0'
+
+ // Unit tests in the local JVM with Robolectric
+ // https://developer.android.com/training/testing/unit-testing/local-unit-tests.html
+ // https://github.com/nenick/AndroidStudioAndRobolectric
+ // http://www.vogella.com/tutorials/Robolectric/article.html
+ testCompile 'junit:junit:4.12'
+ testCompile 'org.robolectric:robolectric:3.0-rc3'
+
+ // UI testing with Espresso
+ androidTestCompile 'com.android.support.test:runner:0.3'
+ androidTestCompile 'com.android.support.test:rules:0.3'
+ androidTestCompile 'com.android.support.test.espresso:espresso-core:2.2'
+ androidTestCompile ('com.android.support.test.espresso:espresso-contrib:2.2') {
+ exclude group: 'com.android.support', module: 'appcompat'
+ exclude group: 'com.android.support', module: 'support-v4'
+ exclude module: 'recyclerview-v7'
+ }
+
+ // Temporary workaround for bug: https://code.google.com/p/android-test-kit/issues/detail?id=136
+ // from https://github.com/googlesamples/android-testing/blob/master/build.gradle#L21
+ configurations.all {
+ resolutionStrategy.force 'com.android.support:support-annotations:22.1.1'
+ }
+
+ // JCenter etc.
+ compile 'com.eftimoff:android-patternview:1.0.1@aar'
+ compile 'com.journeyapps:zxing-android-embedded:2.3.0@aar'
+ compile 'com.journeyapps:zxing-android-integration:2.3.0@aar'
+ compile 'com.google.zxing:core:3.2.0'
+ compile 'com.jpardogo.materialtabstrip:library:1.0.9'
+ compile 'com.getbase:floatingactionbutton:1.9.0'
+ compile 'org.commonjava.googlecode.markdown4j:markdown4j:2.2-cj-1.0'
+ compile 'com.splitwise:tokenautocomplete:1.3.3@aar'
+ compile 'se.emilsjolander:stickylistheaders:2.6.0'
+ compile 'org.sufficientlysecure:html-textview:1.1'
+ compile 'com.mikepenz.materialdrawer:library:2.8.2@aar'
+ compile 'com.mikepenz.iconics:library:0.9.1@aar'
+ compile 'com.mikepenz.iconics:octicons-typeface:2.2.0@aar'
+ compile 'com.mikepenz.iconics:meteocons-typeface:1.1.1@aar'
+ compile 'com.mikepenz.iconics:community-material-typeface:1.0.0@aar'
+ compile 'com.nispok:snackbar:2.10.8'
+
+ // libs as submodules
+ compile project(':extern:openpgp-api-lib:openpgp-api')
+ compile project(':extern:openkeychain-api-lib:openkeychain-intents')
+ compile project(':extern:spongycastle:core')
+ compile project(':extern:spongycastle:pg')
+ compile project(':extern:spongycastle:pkix')
+ compile project(':extern:spongycastle:prov')
+ compile project(':extern:minidns')
+ compile project(':extern:KeybaseLib:Lib')
+ compile project(':extern:safeslinger-exchange')
+}
+
+// Output of ./gradlew -q calculateChecksums
+// Comment out the libs referenced as git submodules!
+dependencyVerification {
+ verify = [
+ 'com.android.support:support-v4:1e2e4d35ac7fd30db5ce3bc177b92e4d5af86acef2ef93e9221599d733346f56',
+ 'com.android.support:appcompat-v7:9a2355537c2f01cf0b95523605c18606b8d824017e6e94a05c77b0cfc8f21c96',
+ 'com.android.support:recyclerview-v7:522d323079a29bcd76173bd9bc7535223b4af3e5eefef9d9287df1f9e54d0c10',
+ 'com.android.support:cardview-v7:8dc99af71fec000baa4470c3907755264f15f816920861bc015b2babdbb49807',
+ 'com.eftimoff:android-patternview:cec80e7265b8d8278b3c55b5fcdf551e4600ac2c8bf60d8dd76adca538af0b1e',
+ 'com.journeyapps:zxing-android-embedded:702a4f58154dbd9baa80f66b6a15410f7a4d403f3e73b66537a8bfb156b4b718',
+ 'com.journeyapps:zxing-android-integration:562737821b6d34c899b6fd2234ce0a8a31e02ff1fd7c59f6211961ce9767c7c8',
+ 'com.google.zxing:core:7fe5a8ff437635a540e56317649937b768b454795ce999ed5f244f83373dee7b',
+ 'com.jpardogo.materialtabstrip:library:c6ef812fba4f74be7dc4a905faa4c2908cba261a94c13d4f96d5e67e4aad4aaa',
+ 'com.getbase:floatingactionbutton:052aa2a94e49e5dccc97cb99f2add87e8698b84859f0e3ac181100c0bc7640ca',
+ 'org.commonjava.googlecode.markdown4j:markdown4j:e952e825d29e1317d96f79f346bfb6786c7c5eef50bd26e54a80823704b62e13',
+ 'com.splitwise:tokenautocomplete:20bee71cc59b3828eb000b684d46ddf738efd56b8fee453a509cd16fda42c8cb',
+ 'se.emilsjolander:stickylistheaders:8c05981ec5725be33f7cee5e68c13f3db49cd5c75f1aaeb04024920b1ef96ad4',
+ 'org.sufficientlysecure:html-textview:ca24b1522be88378634093815ce9ff1b4920c72e7513a045a7846e14069ef988',
+ 'com.mikepenz.materialdrawer:library:970317ed1a3cb96317f7b8d62ff592b3103eb46dfd68d9b244e7143623dc6d7a',
+ 'com.mikepenz.iconics:library:4698a36ee4c2af765d0a85779c61474d755b90d66a59020105b6760a8a909e9e',
+ 'com.mikepenz.iconics:octicons-typeface:67ed7d456a9ce5f5307b85f955797bfb3dd674e2f6defb31c6b8bbe2ede290be',
+ 'com.mikepenz.iconics:meteocons-typeface:39a8a9e70cd8287cdb119af57a672a41dd09240dba6697f5a0dbda1ccc33298b',
+ 'com.mikepenz.iconics:community-material-typeface:f1c5afee5f0f10d66beb3ed0df977246a02a9c46de4e05d7c0264bcde53b6b7f',
+ 'com.nispok:snackbar:80bebc8e5d8b3d728cd5f2336e2d0c1cc2a6b7dc4b55d36acd6b75a78265590a',
+// 'OpenKeychain.extern:openpgp-api-lib:f05a9215cdad3a6597e4c5ece6fcec92b178d218195a3e88d2c0937c48dd9580',
+// 'OpenKeychain.extern:openkeychain-api-lib:50f6ebb5452d3fdc7be137ccf857a0ff44d55539fcb7b91baef495766ed7f429',
+// 'com.madgag.spongycastle:core:df8fcc028a95ac5ffab3b78c9163f5cfa672e41cd50128ca55d458b6cfbacf4b',
+// 'com.madgag.spongycastle:pg:160b345b10a2c92dc731453eec87037377f66a8e14a0648d404d7b193c4e380d',
+// 'com.madgag.spongycastle:pkix:0b4f3301ea12dd9f25d71770e6ea9f75e0611bf53062543e47be5bc15340a7e4',
+// 'com.madgag.spongycastle:prov:7325942e0b39f5fb35d6380818eed4b826e7dfc7570ad35b696d778049d8c36a',
+// 'OpenKeychain.extern:minidns:77b1786d29469e3b21f9404827cab811edc857cd68bc732cd57f11307c332eae',
+// 'OpenKeychain.extern.KeybaseLib:Lib:c91cda4a75692d8664644cd17d8ac962ce5bc0e266ea26673a639805f1eccbdf',
+// 'OpenKeychain.extern:safeslinger-exchange:d222721bb35408daaab9f46449364b2657112705ee571d7532f81cbeb9c4a73f',
+// 'OpenKeychain.extern.snackbar:lib:52357426e5275412e2063bdf6f0e6b957a3ea74da45e0aef35d22d9afc542e23',
+ 'com.android.support:support-annotations:7bc07519aa613b186001160403bcfd68260fa82c61cc7e83adeedc9b862b94ae',
+ ]
+}
+
+android {
+ compileSdkVersion rootProject.ext.compileSdkVersion
+ buildToolsVersion rootProject.ext.buildToolsVersion
+
+ defaultConfig {
+ minSdkVersion 15
+ targetSdkVersion 22
+ versionCode 32300
+ versionName "3.2.3"
+ applicationId "org.sufficientlysecure.keychain"
+ // the androidjunitrunner is broken regarding coverage, see here:
+ // https://code.google.com/p/android/issues/detail?id=170607
+ testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+ // this workaround runner fixes the coverage problem, BUT doesn't work
+ // with android studio single test execution. use it to generate coverage
+ // data, but keep the other one otherwis
+ // testInstrumentationRunner "org.sufficientlysecure.keychain.JacocoWorkaroundJUnitRunner"
+ }
+
+ compileOptions {
+ sourceCompatibility JavaVersion.VERSION_1_7
+ targetCompatibility JavaVersion.VERSION_1_7
+ }
+
+ buildTypes {
+ release {
+ minifyEnabled true
+ proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+
+ // Reference them in the java files with e.g. BuildConfig.ACCOUNT_TYPE.
+ buildConfigField "String", "ACCOUNT_TYPE", "\"org.sufficientlysecure.keychain.account\""
+
+ // Reference them in .xml files.
+ resValue "string", "account_type", "org.sufficientlysecure.keychain.account"
+ }
+
+ debug {
+ applicationIdSuffix ".debug"
+
+ // Reference them in the java files with e.g. BuildConfig.ACCOUNT_TYPE.
+ buildConfigField "String", "ACCOUNT_TYPE", "\"org.sufficientlysecure.keychain.debug.account\""
+
+ // Reference them in .xml files.
+ resValue "string", "account_type", "org.sufficientlysecure.keychain.debug.account"
+
+ // Enable code coverage (Jacoco)
+ testCoverageEnabled true
+ }
+ }
+
+ /*
+ * To sign release build, create file gradle.properties in ~/.gradle/ with this content:
+ *
+ * signingStoreLocation=/home/key.store
+ * signingStorePassword=xxx
+ * signingKeyAlias=alias
+ * signingKeyPassword=xxx
+ */
+ if (project.hasProperty('signingStoreLocation') &&
+ project.hasProperty('signingStorePassword') &&
+ project.hasProperty('signingKeyAlias') &&
+ project.hasProperty('signingKeyPassword')) {
+ println "Found sign properties in gradle.properties! Signing build…"
+
+ signingConfigs {
+ release {
+ storeFile file(signingStoreLocation)
+ storePassword signingStorePassword
+ keyAlias signingKeyAlias
+ keyPassword signingKeyPassword
+ }
+ }
+
+ buildTypes.release.signingConfig = signingConfigs.release
+ } else {
+ buildTypes.release.signingConfig = null
+ }
+
+ // NOTE: Lint is disabled because it slows down builds,
+ // to enable it comment out the code at the bottom of this build.gradle
+ lintOptions {
+ // Do not abort build if lint finds errors
+ abortOnError false
+
+ checkAllWarnings true
+ htmlReport true
+ htmlOutput file('lint-report.html')
+ }
+
+ // Disable preDexing, causes com.android.dx.cf.iface.ParseException: bad class file magic (cafebabe) or version (0034.0000) on some systems
+ dexOptions {
+ preDexLibraries = false
+ }
+
+ packagingOptions {
+ exclude 'LICENSE.txt'
+ }
+}
+
+// apply plugin: 'spoon'
+
+task jacocoTestReport(type:JacocoReport) {
+ group = "Reporting"
+ description = "Generate Jacoco coverage reports"
+
+ classDirectories = fileTree(
+ dir: "${buildDir}/intermediates/classes/debug",
+ excludes: ['**/R.class',
+ '**/R$*.class',
+ '**/*$ViewInjector*.*',
+ '**/BuildConfig.*',
+ '**/Manifest*.*']
+ )
+
+ sourceDirectories = files("${buildDir.parent}/src/main/java")
+ additionalSourceDirs = files([
+ "${buildDir}/generated/source/buildConfig/debug",
+ "${buildDir}/generated/source/r/debug"
+ ])
+ executionData = files([
+ "${buildDir}/jacoco/testDebug.exec",
+ "${buildDir}/outputs/code-coverage/connected/coverage.ec"
+ ])
+
+ reports {
+ xml.enabled = true
+ html.enabled = true
+ }
+}
+
+// Fix for: No report file available: [/home/travis/build/open-keychain/open-keychain/OpenKeychain/build/reports/cobertura/coverage.xml, /home/travis/build/open-keychain/open-keychain/OpenKeychain/build/reports/jacoco/test/jacocoTestReport.xml]
+coveralls {
+ jacocoReportPath 'build/reports/jacoco/jacocoTestReport/jacocoTestReport.xml'
+}
+
+// NOTE: This disables Lint!
+tasks.whenTaskAdded { task ->
+ if (task.name.contains('lint')) {
+ task.enabled = false
+ }
+}
+
--- /dev/null
+buildscript {
+ repositories {
+ jcenter()
+ }
+
+ dependencies {
+ // NOTE: Always use fixed version codes not dynamic ones, e.g. 0.7.3 instead of 0.7.+, see README for more information
+ classpath 'com.android.tools.build:gradle:1.2.3'
+ classpath files('gradle-witness.jar')
+ // bintray dependency to satisfy dependency of openpgp-api lib
+ classpath 'com.novoda:bintray-release:0.2.7'
+
+ classpath 'org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.0.1'
+ // classpath 'com.stanfy.spoon:spoon-gradle-plugin:1.0.2'
+ }
+}
+
+allprojects {
+ repositories {
+ jcenter()
+ }
+}
+
+task wrapper(type: Wrapper) {
+ gradleVersion = '2.4'
+}
+
+subprojects {
+ tasks.withType(Test) {
+ maxParallelForks = 1
+ }
+}
+
+// Ignore tests for external spongycastle
+project(':extern:spongycastle') {
+ subprojects {
+ // Need to re-apply the plugin here otherwise the test property below can't be set.
+ apply plugin: 'java'
+ test.enabled = false
+ }
+}
+
+// SDK Version and Build Tools used by all subprojects
+// See http://tools.android.com/tech-docs/new-build-system/tips#TOC-Controlling-Android-properties-of-all-your-modules-from-the-main-project.
+ext {
+ compileSdkVersion = 22
+ buildToolsVersion = '22.0.1'
+}
--- /dev/null
+apply plugin: 'com.android.application'
+
+// Global Parameters accepted
+// APK_NUMBER_VERSION - version number of apk
+// APK_VERSION - build number like #9999Z, for dev builds appended to app_version like 2.0.0 in no_translate.xml)
+// flavor Z : M=-master, D=-design, B=-Blackberry, MD=-main-default, MQA=-main-qt-arm, MQDA=-main-qt-default-arm, S=-sherpafy
+// TARGET_APP_NAME - app name
+// APP_EDITION - date stamp of builds
+// APP_FEATURES - features +play_market +gps_status -parking_plugin -blackberry -free_version -amazon
+
+
+// 1. To be done Filter fonts
+// <unzip src="OsmAndCore_android.aar" dest=".">
+// <patternset>
+// <include name="assets/**/map/fonts/OpenSans/*"/>
+// <include name="assets/**/map/fonts/NotoSans/*"/>
+// </patternset>
+// </unzip>
+// Less important
+
+android {
+ compileSdkVersion 21
+ buildToolsVersion "21.1.2"
+
+ signingConfigs {
+ development {
+ storeFile file("../keystores/debug.keystore")
+ storePassword "android"
+ keyAlias "androiddebugkey"
+ keyPassword "android"
+ }
+
+ publishing {
+ storeFile file("/var/lib/jenkins/osmand_key")
+ storePassword System.getenv("OSMAND_APK_PASSWORD")
+ keyAlias "osmand"
+ keyPassword System.getenv("OSMAND_APK_PASSWORD")
+ }
+ }
+
+ defaultConfig {
+ minSdkVersion 9
+ targetSdkVersion 21
+
+ versionCode System.getenv("APK_NUMBER_VERSION") ? System.getenv("APK_NUMBER_VERSION").toInteger() : versionCode
+ //versionName already assigned in code
+ //versionName System.getenv("APK_VERSION")? System.getenv("APK_VERSION").toString(): versionName
+ }
+
+ lintOptions {
+ lintConfig file("lint.xml")
+ abortOnError false
+ warningsAsErrors false
+ }
+
+ // This is from OsmAndCore_android.aar - for some reason it's not inherited
+ aaptOptions {
+ // Don't compress any embedded resources
+ noCompress "qz"
+ }
+
+ dexOptions {
+ jumboMode = true
+ }
+
+ sourceSets {
+ main {
+ manifest.srcFile "AndroidManifest.xml"
+ jni.srcDirs = []
+ jniLibs.srcDirs = ["libs"]
+ aidl.srcDirs = ["src"]
+ java.srcDirs = ["src"]
+ resources.srcDirs = ["src"]
+ renderscript.srcDirs = ["src"]
+ res.srcDirs = ["res"]
+ assets.srcDirs = ["assets"]
+ }
+ free {
+ manifest.srcFile "AndroidManifest-free.xml"
+ }
+
+ legacy {
+ jniLibs.srcDirs = ["libgnustl"]
+ }
+ }
+
+ flavorDimensions "version", "coreversion", "abi"
+ productFlavors {
+ // ABI
+ armv7 {
+ flavorDimension "abi"
+ ndk {
+ abiFilter "armeabi-v7a"
+ }
+ }
+ armv5 {
+ flavorDimension "abi"
+ ndk {
+ abiFilter "armeabi"
+ }
+ }
+ x86 {
+ flavorDimension "abi"
+ ndk {
+ abiFilter "x86"
+ }
+ }
+ mips {
+ flavorDimension "abi"
+ ndk {
+ abiFilter "mips"
+ }
+ }
+ fat {
+ flavorDimension "abi"
+ }
+
+ // Version
+ free {
+ flavorDimension "version"
+ applicationId "net.osmand"
+ }
+ full {
+ flavorDimension "version"
+ applicationId "net.osmand.plus"
+ }
+
+ // CoreVersion
+ legacy {
+ flavorDimension "coreversion"
+ }
+
+ qtcore {
+ flavorDimension "coreversion"
+ }
+
+ qtcoredebug {
+ flavorDimension "coreversion"
+ }
+ }
+
+ buildTypes {
+ debug {
+ // proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-project.txt'
+ // minifyEnabled true
+ // proguardFiles 'proguard-project.txt'
+ signingConfig signingConfigs.development
+ }
+ release {
+ // proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-project.txt'
+ // minifyEnabled true
+ //proguardFiles 'proguard-project.txt'
+ signingConfig signingConfigs.publishing
+ }
+ }
+}
+
+def replaceNoTranslate(line) {
+ if (line.contains("\"app_name\"") && System.getenv("TARGET_APP_NAME")) {
+ return line.replaceAll(">[^<]*<", ">" + System.getenv("TARGET_APP_NAME") + "<")
+ }
+ if (line.contains("\"app_edition\"") && System.getenv("APP_EDITION")) {
+ return line.replaceAll(">[^<]*<", ">" + System.getenv("APP_EDITION") + "<")
+ }
+ if (line.contains("\"app_version\"") && System.getenv("APK_VERSION")) {
+ return line.replaceAll(">[^<]*<", ">" + System.getenv("APK_VERSION") + "<")
+ }
+ if (line.contains("\"app_version\"") && System.getenv("APK_VERSION_SUFFIX")) {
+ // appends build number to version number for dev builds
+ return line.replaceAll("</", System.getenv("APK_VERSION_SUFFIX") + "</")
+ }
+ if (line.contains("\"versionFeatures\"") && System.getenv("APP_FEATURES")) {
+ return line.replaceAll(">[^<]*<", ">" + System.getenv("APP_FEATURES") + "<")
+ }
+ return line;
+}
+
+task updateNoTranslate(type: Copy) {
+ from('.') {
+ include 'no_translate.xml'
+ filter {
+ line -> replaceNoTranslate(line);
+ }
+ }
+ into 'res/values/'
+}
+
+task collectVoiceAssets(type: Sync) {
+ from "../../resources/voice"
+ into "assets/voice"
+ include "**/*.p"
+}
+
+task collectHelpContentsAssets(type: Sync) {
+ from "../../help/help"
+ into "assets/help"
+ include "*.html"
+ include "images/**/*.png"
+
+ from "assets/"
+ into "assets/help"
+ include "style.css"
+}
+
+task collectRoutingResources(type: Sync) {
+ from "../../resources/routing"
+ into "src/net/osmand/router"
+ include "*.xml"
+}
+
+task collectMiscResources(type: Copy) {
+ into "src/net/osmand/osm"
+ from("../../resources/obf_creation") {
+ include "rendering_types.xml"
+ }
+ from("../../resources/poi") {
+ include "poi_types.xml"
+ }
+}
+
+task collectRenderingStylesResources(type: Sync) {
+ from "../../resources/rendering_styles"
+ into "src/net/osmand/render"
+ include "*.xml"
+}
+
+task collectRegionsInfoResources(type: Copy) {
+ from "../../resources/countries-info"
+ into "src/net/osmand/map"
+ include "regions.ocbf"
+}
+
+task copyStyleIcons(type: Copy) {
+ from "../../resources/rendering_styles/style-icons/"
+ into "res/"
+ include "**/*.png"
+}
+
+task collectExternalResources << {}
+collectExternalResources.dependsOn collectVoiceAssets,
+ collectHelpContentsAssets,
+ collectRoutingResources,
+ collectRenderingStylesResources,
+ collectRegionsInfoResources,
+ collectMiscResources,
+ copyStyleIcons,
+ updateNoTranslate
+// tasks.whenTaskAdded { task ->
+// if (task.name.startsWith("generate") && task.name.endsWith("Resources")) {
+// task.dependsOn collectExternalResources
+// }
+// }
+
+// Legacy core build
+import org.apache.tools.ant.taskdefs.condition.Os
+
+task buildOsmAndCore(type: Exec) {
+ description "Build Legacy OsmAndCore"
+
+ if (!Os.isFamily(Os.FAMILY_WINDOWS)) {
+ commandLine "bash", file("./old-ndk-build.sh").getAbsolutePath()
+ } else {
+ commandLine "cmd", "/c", "echo", "Not supported"
+ }
+}
+
+task cleanupDuplicatesInCore() {
+ dependsOn buildOsmAndCore
+ // doesn't work for legacy debug builds
+ doLast {
+ file("libgnustl/armeabi").mkdirs()
+ file("libs/armeabi/libgnustl_shared.so").renameTo(file("libgnustl/armeabi/libgnustl_shared.so"))
+ file("libgnustl/armeabi-v7a").mkdirs()
+ file("libs/armeabi-v7a/libgnustl_shared.so").renameTo(file("libgnustl/armeabi-v7a/libgnustl_shared.so"))
+ file("libgnustl/mips").mkdirs()
+ file("libs/mips/libgnustl_shared.so").renameTo(file("libgnustl/mips/libgnustl_shared.so"))
+ file("libgnustl/x86").mkdirs()
+ file("libs/x86/libgnustl_shared.so").renameTo(file("libgnustl/x86/libgnustl_shared.so"))
+ }
+}
+tasks.withType(JavaCompile) {
+ compileTask -> compileTask.dependsOn << [collectExternalResources, buildOsmAndCore, cleanupDuplicatesInCore]
+}
+
+clean.dependsOn 'cleanNoTranslate'
+
+task cleanNoTranslate() {
+ delete ('res/values/no_translate.xml')
+}
+
+repositories {
+ ivy {
+ name = "OsmAndBinariesIvy"
+ url = "http://builder.osmand.net"
+ layout "pattern", {
+ artifact "ivy/[organisation]/[module]/[revision]/[artifact]-[revision].[ext]"
+ }
+ }
+ // mavenCentral()
+}
+
+dependencies {
+ compile project(path: ":OsmAnd-java", configuration: "android")
+ compile project(":eclipse-compile:appcompat")
+ compile fileTree(
+ dir: "libs",
+ include: ["*.jar"],
+ exclude: [
+ "QtAndroid-bundled.jar",
+ "QtAndroidAccessibility-bundled.jar",
+ "OsmAndCore_android.jar",
+ "OsmAndCore_wrapper.jar"])
+ // compile "com.github.ksoichiro:android-observablescrollview:1.5.0"
+ // compile "com.android.support:appcompat-v7:21.0.3"
+ // compile "com.github.shell-software:fab:1.0.5"
+ legacyCompile "net.osmand:OsmAndCore_android:0.1-SNAPSHOT@jar"
+ qtcoredebugCompile "net.osmand:OsmAndCore_androidNativeDebug:0.1-SNAPSHOT@aar"
+ qtcoredebugCompile "net.osmand:OsmAndCore_android:0.1-SNAPSHOT@aar"
+ qtcoreCompile "net.osmand:OsmAndCore_androidNativeRelease:0.1-SNAPSHOT@aar"
+ qtcoreCompile "net.osmand:OsmAndCore_android:0.1-SNAPSHOT@aar"
+}
--- /dev/null
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import sys
+import unittest
+
+localmodule = os.path.realpath(os.path.join(
+ os.path.dirname(inspect.getfile(inspect.currentframe())),
+ '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+ sys.path.insert(0,localmodule)
+
+import fdroidserver.common
+import fdroidserver.update
+from fdroidserver.common import FDroidPopen
+
+class UpdateTest(unittest.TestCase):
+ '''fdroid update'''
+
+ def javagetsig(self, apkfile):
+ getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
+ if not os.path.exists(getsig_dir + "/getsig.class"):
+ logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
+ sys.exit(1)
+ p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
+ 'getsig', os.path.join(os.getcwd(), apkfile)])
+ sig = None
+ for line in p.output.splitlines():
+ if line.startswith('Result:'):
+ sig = line[7:].strip()
+ break
+ if p.returncode == 0:
+ return sig
+ else:
+ return None
+
+ def testGoodGetsig(self):
+ apkfile = os.path.join(os.path.dirname(__file__), 'urzip.apk')
+ sig = self.javagetsig(apkfile)
+ self.assertIsNotNone(sig, "sig is None")
+ pysig = fdroidserver.update.getsig(apkfile)
+ self.assertIsNotNone(pysig, "pysig is None")
+ self.assertEquals(sig, fdroidserver.update.getsig(apkfile),
+ "python sig not equal to java sig!")
+ self.assertEquals(len(sig), len(pysig),
+ "the length of the two sigs are different!")
+ try:
+ self.assertEquals(sig.decode('hex'), pysig.decode('hex'),
+ "the length of the two sigs are different!")
+ except TypeError as e:
+ print e
+ self.assertTrue(False, 'TypeError!')
+
+ def testBadGetsig(self):
+ apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk')
+ sig = self.javagetsig(apkfile)
+ self.assertIsNone(sig, "sig should be None: " + str(sig))
+ pysig = fdroidserver.update.getsig(apkfile)
+ self.assertIsNone(pysig, "python sig should be None: " + str(sig))
+
+ apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk')
+ sig = self.javagetsig(apkfile)
+ self.assertIsNone(sig, "sig should be None: " + str(sig))
+ pysig = fdroidserver.update.getsig(apkfile)
+ self.assertIsNone(pysig, "python sig should be None: " + str(sig))
+
+
+if __name__ == "__main__":
+ parser = optparse.OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+ newSuite = unittest.TestSuite()
+ newSuite.addTest(unittest.makeSuite(UpdateTest))
+ unittest.main()
$out.=$this->get_app($query_vars);
} else {
$out.='<form name="searchform" action="" method="get">';
- $out.='<p><input name="fdfilter" type="text" value="'.$query_vars['fdfilter'].'" size="30"> ';
+ $out.='<p><input name="fdfilter" type="text" value="'.esc_attr($query_vars['fdfilter']).'" size="30"> ';
$out.='<input type="hidden" name="fdpage" value="1">';
$out.='<input type="submit" value="Search"></p>';
$out.=$this->makeformdata($query_vars);
}
}
function androidversion($sdkLevel) {
- if ($sdkLevel < 1) return null;
switch ($sdkLevel) {
+ case 23: return "6.0";
+ case 22: return "5.1";
+ case 21: return "5.0";
+ case 20: return "4.4W";
case 19: return "4.4";
case 18: return "4.3";
case 17: return "4.2";
case "tracker":
$issues=$el;
break;
+ case "changelog":
+ $changelog=$el;
+ break;
case "donate":
$donate=$el;
break;
$out.='<b>Issue Tracker:</b> <a href="'.$issues.'">'.$issues.'</a><br />';
if(strlen($source)>0)
$out.='<b>Source Code:</b> <a href="'.$source.'">'.$source.'</a><br />';
+ if(strlen($changelog)>0)
+ $out.='<b>Changelog:</b> <a href="'.$changelog.'">'.$changelog.'</a><br />';
if(isset($donate) && strlen($donate)>0)
$out.='<b>Donate:</b> <a href="'.$donate.'">'.$donate.'</a><br />';
if(isset($flattr) && strlen($flattr)>0)
$out.='</form>'."\n";
}
else {
- $out.='Applications matching "'.$query_vars['fdfilter'].'"';
+ $out.='Applications matching "'.esc_attr($query_vars['fdfilter']).'"';
}
$out.="</div>";
$out.='<input type="hidden" name="page_id" value="'.(int)get_query_var('page_id').'">';
foreach($query_vars as $name => $value) {
if($value !== null && $name != 'fdfilter' && $name != 'fdpage')
- $out.='<input type="hidden" name="'.$name.'" value="'.sanitize_text_field($value).'">';
+ $out.='<input type="hidden" name="'.esc_attr($name).'" value="'.esc_attr($value).'">';
}
return $out;
~ianmdlvl / fdroidserver.git / commitdiff