Merge pull request khornberg#96 from khornberg/feature/api-gateway-uploads

Version: 3.1.0 -> 3.2.0

Author: khornberg

docs/index.html

@@ -103,6 +103,7 @@ <h2 id="configuration">Configuration</h2>
     TABLE: "elasticpypi"              # You can change me if you want, but do you?
     USERNAME: "elasticpypi"           # You can change me if you want, but do you?
     PASSWORD: "something-secretive"   # CHANGE ME
+    OVERWRITE: false                  # Allow uploads to overwrite already existing packages
 </code></pre><p>
 
         <h2 id="install_dependencies">Install dependencies</h2>

elasticpypi/api.py

@@ -2,30 +2,41 @@
 from flask import Flask
 from flask import send_file
 from flask import render_template
+from flask import request
+from flask import abort
 from elasticpypi.auth import requires_auth
 from elasticpypi import s3, dynamodb
 from elasticpypi.config import config
 
 app = Flask(__name__)
 
 
-@app.route("/simple/")
+@app.route('/simple/', methods=['GET', 'POST'])
 @requires_auth
 def simple():
+    if request.method == 'POST':
+        f = request.files['content']
+        if '/' in f.filename:
+            abort(400)
+        print(config)
+        if config['overwrite'] == 'false' and s3.exists(f.filename):
+            abort(409)
+        s3.upload(f.filename, f.stream)
+        return '', 200
     db = boto3.resource('dynamodb')
     prefixes = dynamodb.list_packages(db)
     return render_template('simple.html', prefixes=prefixes, stage=config['stage'])
 
 
-@app.route("/simple/<name>/")
+@app.route('/simple/<name>/')
 @requires_auth
 def simple_name(name):
     db = boto3.resource('dynamodb')
     packages = dynamodb.list_packages_by_name(db, name)
     return render_template('links.html', packages=packages, package=name, stage=config['stage'])
 
 
-@app.route("/packages/<name>")
+@app.route('/packages/<name>')
 @requires_auth
 def packages_name(name):
     fp = s3.download(name)

package.json

@@ -7,7 +7,7 @@
   },
   "name": "elasticpypi",
   "description": "elastic pypi",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "main": "index.js",
   "directories": {
     "test": "tests"

pytest.ini

@@ -7,3 +7,4 @@ env =
     TABLE=elasticpypi
     USERNAME=elasticpypi
     PASSWORD=something-secretive
+    OVERWRITE=false

readme.md

@@ -1,14 +1,10 @@
 elastic pypi
 ------------
 
-A *mostly* functional simple pypi service running on AWS.
+A fully functional, self-hosted  simple pypi service running on AWS.
 
 # Caveats
 
-**`python setup.py sdist` does not work**
-
-This is a limitation of AWS API Gateway. There are numerous other ways to upload your packages to the S3 bucket.
-
 **Cannot browse with a browser**
 
 This is a limitation of current browsers. They have removed basic authentication for remote urls via the url (e.g. x:y@z). `WWW-Authenticate` responses do not work with AWS Lambda.
@@ -39,6 +35,7 @@ provider:
     TABLE: "elasticpypi"              # You can change me if you want, but do you?
     USERNAME: "elasticpypi"           # You can change me if you want, but do you?
     PASSWORD: "something-secretive"   # CHANGE ME
+    OVERWRITE: false                  # Allow uploads to overwrite already existing packages
 ```
 
 # Deploy
@@ -85,6 +82,8 @@ The example below runs the full test suite. To debug, add `/bin/bash` to the end
 
 # Changelog
 
+* 2017-12-27 Uploads work. Manually tested with `python setup.py upload` and `twine upload`
+
 * *2017-12-22* Use Python 3, downloads go through the API Gateway so pip's caching now works
 
 * *2017-03-24* The configuration has moved from `./elasticpypi/config.json` to `./serverless.yml` and is consumed by elasticpypi as environment variables. If you are upgrading from an older version, you may need to migrate your configuration to serverless.yml.

serverless.yml

@@ -5,16 +5,17 @@ provider:
   runtime: python3.6
   memorySize: 128
   stage: dev
-  # profile: "some-local-aws-config-profile"
+  # profile: 'some-local-aws-config-profile'
   # region: us-east-1
 
   environment:
     SERVICE: ${self:service}
-    STAGE: "/${self:provider.stage}"
-    BUCKET: "elasticpypi"             # CHANGE ME
-    TABLE: "elasticpypi"
-    USERNAME: "elasticpypi"
-    PASSWORD: "something-secretive"   # CHANGE ME
+    STAGE: '/${self:provider.stage}'
+    BUCKET: 'elasticpypi'             # CHANGE ME
+    TABLE: 'elasticpypi'
+    USERNAME: 'elasticpypi'
+    PASSWORD: 'something-secretive'   # CHANGE ME
+    OVERWRITE: false
 
 plugins:
   - serverless-wsgi
@@ -79,27 +80,27 @@ resources:
         TableName: ${self:provider.environment.TABLE}
         AttributeDefinitions:
           -
-            AttributeName: "package_name"
+            AttributeName: 'package_name'
             AttributeType: S
           -
-            AttributeName: "version"
+            AttributeName: 'version'
             AttributeType: S
           -
-            AttributeName: "normalized_name"
+            AttributeName: 'normalized_name'
             AttributeType: S
         KeySchema:
           -
-            AttributeName: "package_name"
+            AttributeName: 'package_name'
             KeyType: HASH
           -
-            AttributeName: "version"
+            AttributeName: 'version'
             KeyType: RANGE
         GlobalSecondaryIndexes:
           -
-            IndexName: "normalized_name-index"
+            IndexName: 'normalized_name-index'
             KeySchema:
               -
-                AttributeName: "normalized_name"
+                AttributeName: 'normalized_name'
                 KeyType: HASH
             Projection:
                 ProjectionType: ALL
@@ -139,5 +140,11 @@ resources:
             - Effect: Allow
               Action: s3:GetObject
               Resource: arn:aws:s3:::${self:provider.environment.BUCKET}/*
+            - Effect: Allow
+              Action: s3:ListBucket
+              Resource: arn:aws:s3:::${self:provider.environment.BUCKET}
+            - Effect: Allow
+              Action: s3:PutObject
+              Resource: arn:aws:s3:::${self:provider.environment.BUCKET}/*
         Roles:
           - Ref: IamRoleLambdaExecution

tests/test_simple.py

@@ -1,3 +1,5 @@
+import io
+from unittest import mock
 from flask_testing import TestCase
 from base64 import b64encode
 from elasticpypi.api import app
@@ -49,3 +51,48 @@ def test_get_simple_200_from_dynamodb(self):
         response = self.client.get('/simple/', headers=self.headers)
         self.assert200(response)
         self.assertEqual(response.data.decode(), fixtures.simple_html)
+
+    def test_post_simple_401(self):
+        response = self.client.post('/simple/')
+        self.assert401(response)
+
+    @mock.patch('elasticpypi.s3.exists')
+    @mock.patch('elasticpypi.s3.upload')
+    def test_post_simple_200(self, upload, exists):
+        exists.return_value = False
+        f = io.BytesIO('hello'.encode('utf-8'))
+        response = self.client.post('/simple/', headers=self.headers, data={'content': (f, 'py-0.1.2.tar.gz')})
+        self.assertStatus(response, 200)
+        upload.assert_called_with('py-0.1.2.tar.gz', mock.ANY)
+        f.close()
+
+    @mock.patch('elasticpypi.s3.upload')
+    def test_cannot_post_file_with_slash_in_the_file_name(self, upload):
+        f = io.BytesIO('hello'.encode('utf-8'))
+        response = self.client.post('/simple/', headers=self.headers, data={'content': (f, '../py-0.1.2.tar.gz')})
+        self.assert400(response)
+        assert not upload.called
+        f.close()
+
+    @mock.patch('elasticpypi.s3.list_packages')
+    @mock.patch('elasticpypi.s3.upload')
+    def test_cannot_post_file_when_package_already_exists_and_overwrite_is_false(self, upload, list_packages):
+        list_packages.return_value = ['py-0.1.2.tar.gz']
+        f = io.BytesIO('hello'.encode('utf-8'))
+        response = self.client.post('/simple/', headers=self.headers, data={'content': (f, 'py-0.1.2.tar.gz')})
+        self.assertEqual(response.status_code, 409)
+        list_packages.assert_called_with('py-0.1.2.tar.gz', True)
+        assert not upload.called
+        f.close()
+
+    @mock.patch('elasticpypi.s3.exists')
+    @mock.patch('elasticpypi.s3.upload')
+    @mock.patch('elasticpypi.api.config')
+    def test_can_post_file_when_package_exists_and_overwrite_is_true(self, config, upload, exists):
+        config.return_value = {'OVERWRITE': 'true'}
+        exists.return_value = True
+        f = io.BytesIO('hello'.encode('utf-8'))
+        response = self.client.post('/simple/', headers=self.headers, data={'content': (f, 'py-0.1.2.tar.gz')})
+        self.assertStatus(response, 200)
+        upload.assert_called_with('py-0.1.2.tar.gz', mock.ANY)
+        f.close()