3 # import.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 from argparse import ArgumentParser
26 from configparser import ConfigParser
31 from . import metadata
32 from .exception import FDroidException
35 SETTINGS_GRADLE = re.compile('''include\s+['"]:([^'"]*)['"]''')
38 # Get the repo type and address from the given web page. The page is scanned
39 # in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
40 # when one of these is found it's assumed that's the information we want.
41 # Returns repotype, address, or None, reason
42 def getrepofrompage(url):
44 req = urllib.request.urlopen(url)
45 if req.getcode() != 200:
46 return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
47 page = req.read().decode(req.headers.get_content_charset())
50 m = re.search('data-fetch-url="(.*)"', page)
54 if repo.endswith('.git'):
59 # Works for BitBucket (obsolete)
60 index = page.find('hg clone')
63 repo = page[index + 9:]
64 index = repo.find('<')
66 return (None, _("Error while getting repo address"))
68 repo = repo.split('"')[0]
69 return (repotype, repo)
71 # Works for BitBucket (obsolete)
72 index = page.find('git clone')
75 repo = page[index + 10:]
76 index = repo.find('<')
78 return (None, _("Error while getting repo address"))
80 repo = repo.split('"')[0]
81 return (repotype, repo)
83 return (None, _("No information found.") + page)
90 def get_metadata_from_url(app, url):
93 if not os.path.isdir(tmp_dir):
94 logging.info(_("Creating temporary directory"))
97 # Figure out what kind of project it is...
99 app.WebSite = url # by default, we might override it
100 if url.startswith('git://'):
106 elif url.startswith('https://github.com'):
107 projecttype = 'github'
111 app.IssueTracker = url + '/issues'
113 elif url.startswith('https://gitlab.com/'):
114 projecttype = 'gitlab'
115 # git can be fussy with gitlab URLs unless they end in .git
116 if url.endswith('.git'):
121 app.SourceCode = url + '/tree/HEAD'
122 app.IssueTracker = url + '/issues'
123 elif url.startswith('https://notabug.org/'):
124 projecttype = 'notabug'
125 if url.endswith('.git'):
130 app.IssueTracker = url + '/issues'
132 elif url.startswith('https://bitbucket.org/'):
133 if url.endswith('/'):
135 projecttype = 'bitbucket'
136 app.SourceCode = url + '/src'
137 app.IssueTracker = url + '/issues'
138 # Figure out the repo type and adddress...
139 repotype, repo = getrepofrompage(url)
141 raise FDroidException("Unable to determine vcs type. " + repo)
142 elif url.startswith('https://') and url.endswith('.git'):
149 raise FDroidException("Unable to determine the project type. " +
150 "The URL you supplied was not in one of the supported formats. " +
151 "Please consult the manual for a list of supported formats, " +
152 "and supply one of those.")
154 # Ensure we have a sensible-looking repo address at this point. If not, we
155 # might have got a page format we weren't expecting. (Note that we
156 # specifically don't want git@...)
157 if ((repotype != 'bzr' and (not repo.startswith('http://') and
158 not repo.startswith('https://') and
159 not repo.startswith('git://'))) or
161 raise FDroidException("Repo address '{0}' does not seem to be valid".format(repo))
163 # Get a copy of the source so we can extract some info...
164 logging.info('Getting source from ' + repotype + ' repo at ' + repo)
165 build_dir = os.path.join(tmp_dir, 'importer')
166 if os.path.exists(build_dir):
167 shutil.rmtree(build_dir)
168 vcs = common.getvcs(repotype, repo, build_dir)
169 vcs.gotorevision(options.rev)
170 root_dir = get_subdir(build_dir)
172 app.RepoType = repotype
175 return root_dir, build_dir
182 def get_subdir(build_dir):
184 return os.path.join(build_dir, options.subdir)
186 settings_gradle = os.path.join(build_dir, 'settings.gradle')
187 if os.path.exists(settings_gradle):
188 with open(settings_gradle) as fp:
189 m = SETTINGS_GRADLE.search(fp.read())
191 return os.path.join(build_dir, m.group(1))
198 global config, options
200 # Parse command line...
201 parser = ArgumentParser()
202 common.setup_global_opts(parser)
203 parser.add_argument("-u", "--url", default=None,
204 help=_("Project URL to import from."))
205 parser.add_argument("-s", "--subdir", default=None,
206 help=_("Path to main Android project subdirectory, if not in root."))
207 parser.add_argument("-c", "--categories", default=None,
208 help=_("Comma separated list of categories."))
209 parser.add_argument("-l", "--license", default=None,
210 help=_("Overall license of the project."))
211 parser.add_argument("--rev", default=None,
212 help=_("Allows a different revision (or git branch) to be specified for the initial import"))
213 metadata.add_metadata_arguments(parser)
214 options = parser.parse_args()
215 metadata.warnings_action = options.W
217 config = common.read_config(options)
219 apps = metadata.read_metadata()
221 app.UpdateCheckMode = "Tags"
226 local_metadata_files = common.get_local_metadata_files()
227 if local_metadata_files != []:
228 raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
230 if options.url is None and os.path.isdir('.git'):
231 app.AutoName = os.path.basename(os.getcwd())
234 build = metadata.Build()
235 root_dir = get_subdir(os.getcwd())
236 if os.path.exists('build.gradle'):
237 build.gradle = ['yes']
240 repo = git.repo.Repo(root_dir) # git repo
241 for remote in git.Remote.iter_items(repo):
242 if remote.name == 'origin':
243 url = repo.remotes.origin.url
244 if url.startswith('https://git'): # github, gitlab
245 app.SourceCode = url.rstrip('.git')
248 # repo.head.commit.binsha is a bytearray stored in a str
249 build.commit = binascii.hexlify(bytearray(repo.head.commit.binsha))
250 write_local_file = True
252 root_dir, build_dir = get_metadata_from_url(app, options.url)
253 build = metadata.Build()
255 build.disable = 'Generated by import.py - check/set version fields and commit id'
256 write_local_file = False
258 raise FDroidException("Specify project url.")
260 # Extract some information...
261 paths = common.manifest_paths(root_dir, [])
264 versionName, versionCode, package = common.parse_androidmanifests(paths, app)
266 raise FDroidException(_("Couldn't find package ID"))
268 logging.warn(_("Couldn't find latest version name"))
270 logging.warn(_("Couldn't find latest version code"))
272 spec = os.path.join(root_dir, 'buildozer.spec')
273 if os.path.exists(spec):
274 defaults = {'orientation': 'landscape', 'icon': '',
275 'permissions': '', 'android.api': "18"}
276 bconfig = ConfigParser(defaults, allow_no_value=True)
278 package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
279 versionName = bconfig.get('app', 'version')
282 raise FDroidException(_("No android or kivy project could be found. Specify --subdir?"))
284 # Make sure it's actually new...
286 raise FDroidException("Package " + package + " already exists")
288 # Create a build line...
289 build.versionName = versionName or '?'
290 build.versionCode = versionCode or '?'
292 build.subdir = options.subdir
294 app.License = options.license
295 if options.categories:
296 app.Categories = options.categories
297 if os.path.exists(os.path.join(root_dir, 'jni')):
298 build.buildjni = ['yes']
299 if os.path.exists(os.path.join(root_dir, 'build.gradle')):
300 build.gradle = ['yes']
302 metadata.post_metadata_parse(app)
304 app.builds.append(build)
307 metadata.write_metadata('.fdroid.yml', app)
309 # Keep the repo directory to save bandwidth...
310 if not os.path.exists('build'):
312 if build_dir is not None:
313 shutil.move(build_dir, os.path.join('build', package))
314 with open('build/.fdroidvcs-' + package, 'w') as f:
315 f.write(app.RepoType + ' ' + app.Repo)
317 metadatapath = os.path.join('metadata', package + '.yml')
318 metadata.write_metadata(metadatapath, app)
319 logging.info("Wrote " + metadatapath)
322 if __name__ == "__main__":