server: switch Amazon AWS S3 upload to streaming mode

This keeps memory usage low because it only has to read a chunk at a time
into memory while before it read the whole file into memory before
uploading it.

This also seems to handle setting the permissions ACL better.

diff --git a/fdroidserver/server.py b/fdroidserver/server.py
index 003dc39665d66468fa7e1d647bb8555c64ed0106..2acd21807af74e0ed72776e057cd28af325d8e89 100644 (file)
--- a/fdroidserver/server.py
+++ b/fdroidserver/server.py
@@ -104,11 +104,11 @@ def update_awsbucket(repo_section):
                     extra['content_type'] = 'application/pgp-signature'
                 logging.info(' uploading ' + os.path.relpath(file_to_upload)
                              + ' to s3://' + awsbucket + '/' + object_name)
-                obj = driver.upload_object(file_path=file_to_upload,
-                                           container=container,
-                                           object_name=object_name,
-                                           verify_hash=False,
-                                           extra=extra)
+                with open(file_to_upload, 'rb') as iterator:
+                    obj = driver.upload_object_via_stream(iterator=iterator,
+                                                          container=container,
+                                                          object_name=object_name,
+                                                          extra=extra)
     # delete the remnants in the bucket, they do not exist locally
     while objs:
         object_name, obj = objs.popitem()