initial commit

Author: hridayns

.gitignore

@@ -0,0 +1,4 @@
+*.sublime-project
+*.sublime-workspace
+.vagrant
+*.txt

Vagrantfile

@@ -0,0 +1,71 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://atlas.hashicorp.com/search.
+  config.vm.box = "hackstreet/hadoop"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  config.vm.synced_folder "./shareFiles", "/home/piggyBoi"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  # config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+  #   vb.memory = "1024"
+  # end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
+  # such as FTP and Heroku are also available. See the documentation at
+  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
+  # config.push.define "atlas" do |push|
+  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
+  # end
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  # config.vm.provision "shell", inline: <<-SHELL
+  #   apt-get update
+  #   apt-get install -y apache2
+  # SHELL
+end

shareFiles/pig_script.pig

@@ -0,0 +1,24 @@
+REGISTER /opt/pig-0.17.0/lib/piggybank.jar;
+
+raw_data = LOAD '/home/piggyBoi/sample_log' USING org.apache.pig.piggybank.storage.apachelog.CombinedLogLoader() AS (ipaddr: chararray, logname: chararray, temp0: chararray,time: chararray, temp1: chararray, temp3: chararray, temp4: chararray, temp5: int, temp6: int, page_link: chararray, temp7: chararray);
+
+grpd_by_page = Group raw_data by page_link;
+
+page_hits = FOREACH grpd_by_page GENERATE flatten($0), COUNT($1) as page_count;
+
+STORE page_hits INTO 'parsed_log';
+
+/*
+ordrd_list = ORDER page_hits BY page_count DESC;
+
+Top_page = LIMIT ordrd_list 1;
+
+store Top_page into 'top_page_res' using PigStorage(',');
+
+DEFINE DATE_EXTRACT org.apache.pig.piggybank.evaluation.util.apachelogparser.DateExtractor('dd/MMM/yyyy:HH:mm:ss Z','d/M/y');
+
+grpd = GROUP raw_data BY DATE_EXTRACT(time);
+
+hits_per_day = foreach grpd generate flatten($0),COUNT($1);
+
+store hits_per_day into 'hits_per_day' using PigStorage(',');*/

shareFiles/pig_test_script.pig

@@ -0,0 +1,12 @@
+REGISTER /opt/pig-0.17.0/lib/piggybank.jar;
+REGISTER  'script.py' using jython as myfuncs;
+
+raw_data = LOAD '/home/piggyBoi/chat1.txt' USING PigStorage('\n') AS (line:chararray);
+
+data = FOREACH raw_data GENERATE FLATTEN(myfuncs.customParser(line));
+
+/*grp_by_sender = GROUP data BY sender;*/
+
+grp_by_word_id = GROUP data BY result::words.id;
+
+STORE grp_by_word_id INTO 'parsed_chat';