Implement first integration with PostgreSQL DB

Adjust tests to import data from fixture file `./fixtures/articles_and_authors.json` instead of using mock values defined at `models.authors` and `models.articles`.

Adjust `.gitlab-ci.yml` file in order to require postgres service, configure its variables, and install `libpq-dev` system package which is required by `libpq-dev` python package.

Note, at first I thought Django's `migration` were not meant to be added to VCS. However, according to [Django's documentation](https://docs.djangoproject.com/en/1.7/topics/migrations/) it seems it is:

> The migration files for each app live in a “migrations” directory inside of that app, and are designed to be committed to, and distributed as part of, its codebase. You should be making them once on your development machine and then running the same migrations on your colleagues’ machines, your staging machines, and eventually your production machines.

Author: fcv

.gitlab-ci.yml

@@ -3,7 +3,26 @@
 # - https://docs.gitlab.com/ce/ci/examples/test-and-deploy-python-application-to-heroku.html
 image: python:3.5
 
+services:
+  - postgres:9.3
+
+variables:
+  # Configure postgres service (https://hub.docker.com/_/postgres/)
+  # based on https://gitlab.com/gitlab-examples/postgres/blob/master/.gitlab-ci.yml
+  # and http://docs.gitlab.com/ce/ci/services/postgres.html
+  POSTGRES_DB: web_scraper_test
+  POSTGRES_USER: web_scraper_user
+  POSTGRES_PASSWORD: web_scraper_pwd
+
 test:
   script:
+    # installs libpq-dev which is required by psycopg2
+    # avoiding error message like:
+    #     Error: b'You need to install postgresql-server-dev-X.Y for building a server-side extension or libpq-dev for building a client-side application.\n'
+    # see more at https://pypi.python.org/pypi/psycopg2
+    - apt-get -qq update
+    - apt-get -y -qq install libpq-dev
+    # install project's dependencies
     - pip install -r requirements.txt
+    # actually execute tests
     - python3 manage.py test

README.md

@@ -3,3 +3,9 @@
 ## About
 
 Simple Web Scraper POC project. Aims to retrieve data from a Web Page and expose extracted data through a REST API.
+
+## Note about PostgreSQL usage
+
+In ubuntu environment, in order to install PostgreSQL database adapter [psycopg2](https://pypi.python.org/pypi/psycopg2) it is necessary first to install `libpq-dev` package in order to avoid error message like one below:
+
+    Error: b'You need to install postgresql-server-dev-X.Y for building a server-side extension or libpq-dev for building a client-side application.\n'

requirements.txt

@@ -1,3 +1,4 @@
 django==1.8.13
 jsonpath-rw==1.4.0
 djangorestframework==3.4.7
+psycopg2==2.6.2

simple_web_scraper/fixtures/articles_and_authors.json

@@ -0,0 +1 @@
+[{"model": "simple_web_scraper.author", "fields": {"profile_url": "https://techcrunch.com/author/anthony-ha/", "name": "Anthony Ha"}, "pk": 1}, {"model": "simple_web_scraper.author", "fields": {"profile_url": "https://techcrunch.com/author/sarah-perez/", "name": "Sarah Perez"}, "pk": 2}, {"model": "simple_web_scraper.article", "fields": {"url": "https://techcrunch.com/2016/10/06/amazonfresh-drops-to-14-99-per-month-for-prime-members/", "title": "AmazonFresh drops to $14.99 per month for Prime members", "authors": [1, 2], "publish_date": null, "content": "a nice content"}, "pk": 1}]

simple_web_scraper/migrations/0001_initial.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Article',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
+                ('title', models.CharField(max_length=100)),
+                ('url', models.URLField()),
+                ('publish_date', models.DateTimeField(null=True, blank=True)),
+                ('content', models.CharField(max_length=500)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Author',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('profile_url', models.URLField()),
+            ],
+        ),
+        migrations.AddField(
+            model_name='article',
+            name='authors',
+            field=models.ManyToManyField(to='simple_web_scraper.Author'),
+        ),
+    ]

simple_web_scraper/migrations/__init__.py

@@ -75,11 +75,20 @@
 
 # Database
 # https://docs.djangoproject.com/en/1.8/ref/settings/#databases
-
+# https://www.digitalocean.com/community/tutorials/how-to-use-postgresql-with-your-django-application-on-ubuntu-14-04
+# https://www.digitalocean.com/community/tutorials/how-to-use-postgresql-with-your-django-application-on-ubuntu-14-04
+# http://stackoverflow.com/questions/5394331/how-to-setup-postgresql-database-in-django/5421511#5421511
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'web_scraper',
+        'USER': 'web_scraper_user',
+        'PASSWORD': 'web_scraper_pwd',
+        'HOST': 'localhost',
+        'PORT': '5432',
+        'TEST': {
+            'NAME': 'web_scraper_test'
+        }
     }
 }
 

simple_web_scraper/settings.py

@@ -17,14 +17,14 @@ class ArticleList(APIView):
     """
 
     def get(self, request, format = None):
-        articles = models.articles
+        articles = Article.objects.all()
         serializer = ArticleSerializer(articles, many=True)
         return Response(serializer.data)
 
     def post(self, request, format = None):
         serializer = ArticleSerializer(data=request.data)
         if serializer.is_valid():
-            # print(serializer.data)
+            serializer.save()
             return Response(serializer.data, status=status.HTTP_201_CREATED)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
@@ -35,27 +35,28 @@ def get_object(self, id):
             id = int(id)
         except ValueError:
             raise HttpResponseBadRequest()
-        article = next(filter(lambda a: a.id == id, models.articles), None)
-        if article is None:
+        try:
+            article = Article.objects.get(pk = id)
+        except Article.DoesNotExist:
             raise Http404
         return article
 
     def get(self, request, id, format=None):
         article = self.get_object(id)
-        serializer = AuthorSerializer(article)
+        serializer = ArticleSerializer(article)
         return Response(serializer.data)
 
     def put(self, request, id, format=None):
         article = self.get_object(id)
         serializer = ArticleSerializer(article, data=request.data)
         if serializer.is_valid():
-            # print(serializer.data)
+            serializer.save()
             return Response(serializer.data)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
     def delete(self, request, id, format=None):
         article = self.get_object(id)
-        # article.delete()
+        article.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 class AuthorList(APIView):
@@ -69,14 +70,14 @@ class AuthorList(APIView):
     """
 
     def get(self, request, format = None):
-        authors = models.authors
+        authors = Author.objects.all()
         serializer = AuthorSerializer(authors, many=True)
         return Response(serializer.data)
 
     def post(self, request, format = None):
         serializer = AuthorSerializer(data=request.data)
         if serializer.is_valid():
-            # print(serializer.data)
+            serializer.save()
             return Response(serializer.data, status=status.HTTP_201_CREATED)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
@@ -88,8 +89,9 @@ def get_object(self, id):
             id = int(id)
         except ValueError:
             raise HttpResponseBadRequest()
-        author = next(filter(lambda a: a.id == id, models.authors), None)
-        if author is None:
+        try:
+            author = Author.objects.get(pk = id)
+        except Author.DoesNotExist:
             raise Http404
         return author
 
@@ -102,11 +104,11 @@ def put(self, request, id, format=None):
         author = self.get_object(id)
         serializer = AuthorSerializer(author, data=request.data)
         if serializer.is_valid():
-            # print(serializer.data)
+            serializer.save()
             return Response(serializer.data)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
     def delete(self, request, id, format=None):
         author = self.get_object(id)
-        # author.delete()
+        author.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)

simple_web_scraper/views.py

@@ -1,10 +1,11 @@
-import unittest
-from django.test import Client
+from django.test import Client, TransactionTestCase
 import json
 from jsonpath_rw import jsonpath, parse
 
-class ArticleEndpointTest(unittest.TestCase):
 
+class ArticleEndpointTest(TransactionTestCase):
+
+    fixtures = ['articles_and_authors.json']
     endpoint = '/api/rest/v1/articles'
 
     def test_list(self):
@@ -17,7 +18,7 @@ def test_list(self):
         # see more about jsonpath at http://goessner.net/articles/JsonPath/
         # and its python port at https://github.com/kennknowles/python-jsonpath-rw
         actual_ids = [match.value for match in parse('$.[*].id').find(content_json)]
-        self.assertEquals([3], actual_ids)
+        self.assertEquals([1], actual_ids)
 
     def test_404_when_id_doesnt_exist(self):
         client = Client()
@@ -27,7 +28,6 @@ def test_404_when_id_doesnt_exist(self):
     def test_post(self):
         client = Client()
         json_str = json.dumps({
-            "id": 1,
             "title": "a nice title",
             "url": "http://johndoo.org",
             "content": "dummy content",
@@ -37,7 +37,7 @@ def test_post(self):
 
     def test_delete(self):
         client = Client()
-        response = client.delete(ArticleEndpointTest.endpoint + '/3')
+        response = client.delete(ArticleEndpointTest.endpoint + '/1')
         self.assertEqual(response.status_code, 204)
 
     def test_delete_not_found(self):

test/test_article_endpoint.py

@@ -1,11 +1,11 @@
-import unittest
-from django.test import Client
+from django.test import Client, TransactionTestCase
 import json
 from jsonpath_rw import jsonpath, parse
 
 
-class AuthorEndpointTest(unittest.TestCase):
+class AuthorEndpointTest(TransactionTestCase):
 
+    fixtures = ['articles_and_authors.json']
     endpoint = '/api/rest/v1/authors'
 
     def test_list(self):