from datetime import datetime
from xml.dom.minidom import Document
-import requests
-
-from fdroidserver import metadata, signindex, common
+from fdroidserver import metadata, signindex, common, net
from fdroidserver.common import FDroidPopen, FDroidPopenBytes
from fdroidserver.metadata import MetaDataException
pass
-def download_repo_index(url_str, verify_fingerprint=True):
+def download_repo_index(url_str, etag=None, verify_fingerprint=True):
"""
Downloads the repository index from the given :param url_str
and verifies the repository's fingerprint if :param verify_fingerprint is not False.
:raises: VerificationException() if the repository could not be verified
- :return: The index in JSON format.
+ :return: A tuple consisting of:
+ - The index in JSON format or None if the index did not change
+ - The new eTag as returned by the HTTP request
"""
url = urllib.parse.urlsplit(url_str)
fingerprint = query['fingerprint'][0]
url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
- r = requests.get(url.geturl())
+ download, new_etag = net.http_get(url.geturl(), etag)
+
+ if download is None:
+ return None, new_etag
with tempfile.NamedTemporaryFile() as fp:
# write and open JAR file
- fp.write(r.content)
+ fp.write(download)
jar = zipfile.ZipFile(fp)
# verify that the JAR signature is valid
# turn the apps into App objects
index["apps"] = [metadata.App(app) for app in index["apps"]]
- return index
+ return index, new_etag
def verify_jar_signature(file):
f.write(chunk)
f.flush()
return local_filename
+
+
+def http_get(url, etag=None):
+ """
+ Downloads the content from the given URL by making a GET request.
+
+ If an ETag is given, it will do a HEAD request first, to see if the content changed.
+
+ :param url: The URL to download from.
+ :param etag: The last ETag to be used for the request (optional).
+ :return: A tuple consisting of:
+ - The raw content that was downloaded or None if it did not change
+ - The new eTag as returned by the HTTP request
+ """
+ headers = {'User-Agent': 'F-Droid'}
+ # TODO disable TLS Session IDs and TLS Session Tickets
+ # (plain text cookie visible to anyone who can see the network traffic)
+ if etag:
+ r = requests.head(url, headers=headers)
+ r.raise_for_status()
+ if 'ETag' in r.headers and etag == r.headers['ETag']:
+ return None, etag
+
+ r = requests.get(url, headers=headers)
+ r.raise_for_status()
+
+ new_etag = None
+ if 'ETag' in r.headers:
+ new_etag = r.headers['ETag']
+
+ return r.content, new_etag
import sys
import unittest
import zipfile
+from unittest.mock import patch
+
+import requests
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
import fdroidserver.signindex
+GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
+
+
class IndexTest(unittest.TestCase):
def setUp(self):
'818E469465F96B704E27BE2FEE4C63AB' +
'9F83DDF30E7A34C7371A4728D83B0BC1')
if f == 'guardianproject.jar':
- self.assertTrue(fingerprint ==
- 'B7C2EEFD8DAC7806AF67DFCD92EB1812' +
- '6BC08312A7F2D6F3862E46013C7A6135')
+ self.assertTrue(fingerprint == GP_FINGERPRINT)
def test_get_public_key_from_jar_fails(self):
basedir = os.path.dirname(__file__)
fdroidserver.index.download_repo_index("http://example.org")
def test_download_repo_index_no_jar(self):
- with self.assertRaises(zipfile.BadZipFile):
+ with self.assertRaises(requests.exceptions.HTTPError):
fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")
- # TODO test_download_repo_index with an actual repository
+ @patch('requests.head')
+ def test_download_repo_index_same_etag(self, head):
+ url = 'http://example.org?fingerprint=test'
+ etag = '"4de5-54d840ce95cb9"'
+
+ head.return_value.headers = {'ETag': etag}
+ index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
+
+ self.assertIsNone(index)
+ self.assertEqual(etag, new_etag)
+
+ @patch('requests.get')
+ @patch('requests.head')
+ def test_download_repo_index_new_etag(self, head, get):
+ url = 'http://example.org?fingerprint=' + GP_FINGERPRINT
+ etag = '"4de5-54d840ce95cb9"'
+
+ # fake HTTP answers
+ head.return_value.headers = {'ETag': 'new_etag'}
+ get.return_value.headers = {'ETag': 'new_etag'}
+ get.return_value.status_code = 200
+ testfile = os.path.join(os.path.dirname(__file__), 'signindex', 'guardianproject-v1.jar')
+ with open(testfile, 'rb') as file:
+ get.return_value.content = file.read()
+
+ index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
+
+ # assert that the index was retrieved properly
+ self.assertEqual('Guardian Project Official Releases', index['repo']['name'])
+ self.assertEqual(GP_FINGERPRINT, index['repo']['fingerprint'])
+ self.assertTrue(len(index['repo']['pubkey']) > 500)
+ self.assertEqual(10, len(index['apps']))
+ self.assertEqual(10, len(index['packages']))
+ self.assertEqual('new_etag', new_etag)
if __name__ == "__main__":