twitterScrapping

Readme.md

@@ -13,3 +13,7 @@ pip install django-cors-headers
 pip install djangorestframework-simplejwt
 
 pip install dj_database_url
+
+pip install tweepy
+
+pip install feedparser 

config.py

@@ -0,0 +1,7 @@
+#for twitter scrapping
+CONSUMER_KEY = 'dF8LarLqWcQAlZ3FqomAFGWNZ'
+CONSUMER_SECRET = 'eaKxAM5scchwogxYo8oEzC4XkangtrDMsuNqx8ZlqBDuVMpBpZ'
+ACCESS_TOKEN = '1046391379360919552-IOEg7xCFH03syq18cGhU3DJxDRVj4D'
+ACCESS_TOKEN_SECRET = 'UUjxB6r8yyEQAe0ffcGI7oqPvKI50pxwwmZGSmXkbQbt3'
+
+#email de notifications

requirements.txt

@@ -8,5 +8,5 @@ djangorestframework
 gunicorn
 psycopg2-binary
 dj-database-url
-Scrapy
+tweepy
 feedparser

robot/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.0.4 on 2020-06-22 13:56
+# Generated by Django 3.0.4 on 2020-06-25 21:45
 
 from django.db import migrations, models
 
@@ -12,35 +12,41 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.CreateModel(
-            name='GoogleSearchResult',
+            name='Tweets',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('proprio', models.CharField(max_length=50)),
+                ('content', models.CharField(max_length=500)),
+                ('date', models.DateTimeField()),
                 ('valide', models.BooleanField(default=False)),
                 ('supprime', models.BooleanField(default=False)),
-                ('titre', models.CharField(max_length=20)),
-                ('description', models.TextField()),
                 ('url', models.URLField()),
-                ('date', models.DateTimeField()),
             ],
         ),
         migrations.CreateModel(
-            name='Tweets',
+            name='VideoYoutube',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('valide', models.BooleanField(default=False)),
                 ('supprime', models.BooleanField(default=False)),
+                ('titre', models.CharField(max_length=200)),
+                ('description', models.TextField()),
                 ('url', models.URLField()),
             ],
         ),
         migrations.CreateModel(
-            name='VideoYoutube',
+            name='GoogleSearchResult',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('valide', models.BooleanField(default=False)),
                 ('supprime', models.BooleanField(default=False)),
-                ('titre', models.CharField(max_length=20)),
+                ('titre', models.CharField(max_length=200)),
                 ('description', models.TextField()),
                 ('url', models.URLField()),
+                ('date', models.CharField(max_length=50)),
             ],
+            options={
+                'unique_together': {('url',)},
+            },
         ),
     ]

robot/migrations/0002_remove_tweets_url.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.0.4 on 2020-06-25 21:49
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('robot', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='tweets',
+            name='url',
+        ),
+    ]

robot/migrations/0002_auto_20200622_1632.py → robot/migrations/0003_auto_20200625_2356.py

@@ -1,17 +1,17 @@
-# Generated by Django 3.0.4 on 2020-06-22 14:32
+# Generated by Django 3.0.4 on 2020-06-25 21:56
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('robot', '0001_initial'),
+        ('robot', '0002_remove_tweets_url'),
     ]
 
     operations = [
         migrations.AlterField(
-            model_name='googlesearchresult',
+            model_name='tweets',
             name='date',
             field=models.CharField(max_length=50),
         ),

robot/models.py

@@ -4,19 +4,22 @@
 class VideoYoutube(models.Model):
     valide = models.BooleanField(default=False)
     supprime = models.BooleanField(default=False)
-    titre = models.CharField(max_length=20)
+    titre = models.CharField(max_length=200)
     description = models.TextField()
     url= models.URLField()
 
 class Tweets(models.Model):
+    proprio=models.CharField(max_length=50)
+    content= models.CharField(max_length=500)
+    date= models.CharField(max_length=50)
     valide = models.BooleanField(default=False)
     supprime = models.BooleanField(default=False)
-    url = models.URLField()
+
 
 class GoogleSearchResult(models.Model):
     valide = models.BooleanField(default=False)
     supprime = models.BooleanField(default=False)
-    titre = models.CharField(max_length=20)
+    titre = models.CharField(max_length=200)
     description = models.TextField()
     url= models.URLField()
     date= models.CharField(max_length=50)

robot/twitterScrapping.py

@@ -0,0 +1,22 @@
+import tweepy
+import sys
+from .models import Tweets
+from config import ACCESS_TOKEN, ACCESS_TOKEN_SECRET,CONSUMER_KEY,CONSUMER_SECRET
+
+def getTweets(counts):
+
+    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
+    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
+
+    api = tweepy.API(auth)
+    try:
+        api.verify_credentials()
+        print('Authentication Successful')
+    except:
+        print('Error while authenticating API')
+        sys.exit(1)
+
+    covid_tweets = tweepy.Cursor(api.search, q='فيروس كورونا').items(counts)
+    for tweet in covid_tweets:
+        tweetObj= Tweets.objects.create(content=tweet.text, date=tweet.created_at, proprio=tweet.user.name)
+        tweetObj.save()

robot/views.py

@@ -6,7 +6,7 @@
 
 
 from rest_framework import viewsets, permissions
-
+from .twitterScrapping import getTweets
 from .googleSearchScrapping import ParseFeed
 from .serializers import *
 from .models import *
@@ -119,8 +119,9 @@ class TweetsViewSet(viewsets.ModelViewSet):
     serializer_class = TweetSerializer
 
     @action(methods=['post', 'get'], detail=False)
-    def show_list(self, request):
+    def getData(self, request):
         if(request.method == "GET"):
+            getTweets(50)
             data = Tweets.objects.all()
             serializers = TweetSerializer(data, many=True)
             return Response(serializers.data)