server: check virustotal has APK before uploading it

This restructures the virustotal uploading so that first checks whether
virustotal already has the file, and only if not does it upload it.  This
also handles the public API rate limiting, which returns an HTTP 204. This
will now try again until it succeeds, even when rate limited.   Instead of
just getting the list of files from the filesystem, this reads the index-v1
which also already has the SHA256 in there.  virustotal also uses SHA256 as
a unique ID for files.

diff --git a/fdroidserver/server.py b/fdroidserver/server.py
index ae3d8c9a24140f93d3b2a26b1d678df29a7aa21d..5cc5db81660e794d19336b2de2b5446e54cb1188 100644 (file)
--- a/fdroidserver/server.py
+++ b/fdroidserver/server.py
@@ -24,6 +24,7 @@ import paramiko
 import pwd
 import re
 import subprocess
+import time
 from argparse import ArgumentParser
 import logging
 import shutil
@@ -401,21 +402,53 @@ def upload_to_android_observatory(repo_section):
 
 
 def upload_to_virustotal(repo_section, vt_apikey):
+    import json
     import requests
 
-    if repo_section == 'repo':
-        for f in glob.glob(os.path.join(repo_section, '*.apk')):
-            fpath = f
-            fname = os.path.basename(f)
-            logging.info('Uploading ' + fname + ' to virustotal.com')
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
 
-            # upload the file with a post request
-            params = {'apikey': vt_apikey}
-            files = {'file': (fname, open(fpath, 'rb'))}
-            r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=params)
-            response = r.json()
-
-            logging.info(response['verbose_msg'] + " " + response['permalink'])
+    if repo_section == 'repo':
+        with open(os.path.join(repo_section, 'index-v1.json')) as fp:
+            index = json.load(fp)
+        for packageName, packages in index['packages'].items():
+            for package in packages:
+                filename = package['apkName']
+                repofilename = os.path.join(repo_section, filename)
+                logging.info('Uploading ' + repofilename + ' to virustotal.com')
+
+                headers = {
+                    "User-Agent": "F-Droid"
+                }
+                params = {
+                    'apikey': vt_apikey,
+                    'resource': package['hash'],
+                }
+                download = False
+                while True:
+                    r = requests.post('https://www.virustotal.com/vtapi/v2/file/report',
+                                      params=params, headers=headers)
+                    if r.status_code == 200:
+                        response = r.json()
+                        if response['response_code'] == 0:
+                            download = True
+                        elif response['positives'] > 0:
+                            logging.warning(repofilename + ' has been flagged by virustotal '
+                                            + str(response['positives']) + 'times:'
+                                            + '\n\t' + response['permalink'])
+                        break
+                    elif r.status_code == 204:
+                        time.sleep(10)  # wait for public API rate limiting
+
+                if download:
+                    files = {
+                        'file': (filename, open(repofilename, 'rb'))
+                    }
+                    r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan',
+                                      params=params, headers=headers, files=files)
+                    response = r.json()
+
+                    logging.info(response['verbose_msg'] + " " + response['permalink'])
 
 
 def push_binary_transparency(git_repo_path, git_remote):