Skip to content

Commit ba96e7c

Browse files
tannerwelshtannerwelsh
committed
README and code files for 2015/02/14 session
1 parent fc30f55 commit ba96e7c

File tree

6 files changed

+240
-36
lines changed

6 files changed

+240
-36
lines changed

‎session-notes/2015-02-14/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Fundamentals of Web Development
2+
3+
**February 14, 2015**
4+
5+
## Craigslist Scraper
6+
7+
Stage 1: Create our data model & build a relational database to store CL info
8+
9+
Rough sketch of CL data model:
10+
- MetroArea [cities]
11+
- City/Suburb/District [sections]
12+
- Section [categories]
13+
- Category [filter]
14+
- Posting [price, condition, make, model, description, title]
15+
16+
Main query: search for bikes by brand and price
17+
18+
model bike posting
19+
20+
```ruby
21+
# Search for a bike by brand and price
22+
bikes = Craiglist.bikes(brand: 'Santa Cruz', max_price: 400.00)
23+
# => return an array of Craigslist::Posting objects
24+
25+
bikes.first.title == 'My Mountain Bike for Sale'
26+
```

‎session-notes/2015-02-14/craigslist/Gemfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
source 'https://rubygems.org'
2+
3+
gem 'sqlite3'
4+
gem 'data_mapper'

‎session-notes/2015-02-14/craigslist/Gemfile.lock

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
GEM
2+
remote: https://rubygems.org/
3+
specs:
4+
addressable (2.3.7)
5+
bcrypt (3.1.10)
6+
bcrypt-ruby (3.1.5)
7+
bcrypt (>= 3.1.3)
8+
data_mapper (1.2.0)
9+
dm-aggregates (~> 1.2.0)
10+
dm-constraints (~> 1.2.0)
11+
dm-core (~> 1.2.0)
12+
dm-migrations (~> 1.2.0)
13+
dm-serializer (~> 1.2.0)
14+
dm-timestamps (~> 1.2.0)
15+
dm-transactions (~> 1.2.0)
16+
dm-types (~> 1.2.0)
17+
dm-validations (~> 1.2.0)
18+
dm-aggregates (1.2.0)
19+
dm-core (~> 1.2.0)
20+
dm-constraints (1.2.0)
21+
dm-core (~> 1.2.0)
22+
dm-core (1.2.1)
23+
addressable (~> 2.3)
24+
dm-migrations (1.2.0)
25+
dm-core (~> 1.2.0)
26+
dm-serializer (1.2.2)
27+
dm-core (~> 1.2.0)
28+
fastercsv (~> 1.5)
29+
json (~> 1.6)
30+
json_pure (~> 1.6)
31+
multi_json (~> 1.0)
32+
dm-timestamps (1.2.0)
33+
dm-core (~> 1.2.0)
34+
dm-transactions (1.2.0)
35+
dm-core (~> 1.2.0)
36+
dm-types (1.2.2)
37+
bcrypt-ruby (~> 3.0)
38+
dm-core (~> 1.2.0)
39+
fastercsv (~> 1.5)
40+
json (~> 1.6)
41+
multi_json (~> 1.0)
42+
stringex (~> 1.4)
43+
uuidtools (~> 2.1)
44+
dm-validations (1.2.0)
45+
dm-core (~> 1.2.0)
46+
fastercsv (1.5.5)
47+
json (1.8.2)
48+
json_pure (1.8.2)
49+
multi_json (1.10.1)
50+
sqlite3 (1.3.10)
51+
stringex (1.5.1)
52+
uuidtools (2.1.5)
53+
54+
PLATFORMS
55+
ruby
56+
57+
DEPENDENCIES
58+
data_mapper
59+
sqlite3

‎session-notes/2015-02-14/craigslist/craigslist.db

44 KB
Binary file not shown.

‎session-notes/2015-02-14/craigslist/craigslist.rb

Lines changed: 129 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,41 +2,143 @@
22
# 2. Add some hard-coded dummy data
33
# 3. Replace dummy data with database and DataMapper
44

5+
require 'data_mapper'
6+
7+
# Database schema
8+
#
9+
# TABLES:
10+
# postings [title, price, description, category_id]
11+
# categories [title, url_keystring, section_id]
12+
# sections [title, url_keystring, city_id]
13+
# cities [name, url_keystring, metro_area_id]
14+
# metro_areas [name, subdomain]
15+
16+
DataMapper.setup(:default, 'sqlite:craigslist.db')
17+
518
module Craigslist
6-
ALL_POSTINGS = []
19+
def self.bikes
20+
cat = Category.first(title: 'bikes')
21+
cat.postings
22+
end
23+
24+
class Posting
25+
include DataMapper::Resource
726

8-
def self.all
9-
ALL_POSTINGS
27+
property :id, Serial
28+
property :title, String, required: true
29+
property :price, Float
30+
property :description, Text
31+
property :make, String
32+
33+
belongs_to :category
1034
end
1135

12-
# bikes = Craigslist.bikes(make: 'Santa Cruz', max_price: 400.00)
13-
def self.bikes(search_opts)
14-
make = search_opts[:make]
15-
max_price = search_opts[:max_price]
36+
class Category
37+
include DataMapper::Resource
1638

17-
results = self.all.select do |posting|
18-
posting.category == 'bikes' &&
19-
posting.make == make &&
20-
posting.price < max_price
21-
end
39+
property :id, Serial
40+
property :title, String, required: true
41+
property :url_keystring, String, required: true
2242

23-
return results
43+
has n, :postings
44+
# belongs_to :section
2445
end
2546

26-
Craigslist.bikes({})
47+
class Section
48+
include DataMapper::Resource
2749

28-
class Posting
29-
attr_reader :category, :title, :price, :make
30-
31-
def initialize(fields)
32-
@category = fields[:category]
33-
@title = fields[:title]
34-
@price = fields[:price]
35-
@make = fields[:make]
36-
end
37-
38-
def save
39-
ALL_POSTINGS << self
40-
end
50+
property :id, Serial
51+
property :title, String, required: true
52+
property :url_keystring, String, required: true
53+
54+
# has n, :categories
55+
belongs_to :city
56+
end
57+
58+
class City
59+
include DataMapper::Resource
60+
61+
property :id, Serial
62+
property :name, String, required: true
63+
property :url_keystring, String, required: true
64+
65+
has n, :sections
66+
belongs_to :metro_area
67+
end
68+
69+
class MetroArea
70+
include DataMapper::Resource
71+
72+
property :id, Serial
73+
property :name, String, required: true
74+
property :subdomain, String, required: true
75+
76+
has n, :cities
4177
end
4278
end
79+
80+
DataMapper.finalize
81+
DataMapper.auto_migrate!
82+
83+
84+
85+
86+
87+
88+
89+
90+
91+
92+
93+
94+
95+
96+
97+
98+
99+
100+
101+
102+
103+
104+
105+
106+
107+
108+
#
109+
# module Craigslist
110+
# ALL_POSTINGS = []
111+
#
112+
# def self.all
113+
# ALL_POSTINGS
114+
# end
115+
#
116+
# # bikes = Craigslist.bikes(make: 'Santa Cruz', max_price: 400.00)
117+
# def self.bikes(search_opts)
118+
# make = search_opts[:make]
119+
# max_price = search_opts[:max_price]
120+
#
121+
# results = self.all.select do |posting|
122+
# posting.category == 'bikes' &&
123+
# posting.make == make &&
124+
# posting.price < max_price
125+
# end
126+
#
127+
# return results
128+
# end
129+
#
130+
# class Posting
131+
# attr_reader :category, :title, :price, :make
132+
#
133+
# def initialize(fields)
134+
# @category = fields[:category]
135+
# @title = fields[:title]
136+
# @price = fields[:price]
137+
# @make = fields[:make]
138+
# end
139+
#
140+
# def save
141+
# ALL_POSTINGS << self
142+
# end
143+
# end
144+
# end

‎session-notes/2015-02-14/craigslist/craigslist_test.rb

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,52 @@
22

33
# TESTS ---------------------------------------
44

5+
bikes_cat = Craigslist::Category.new(title: "bikes", url_keystring: 'bia')
6+
7+
p bikes_cat.valid?
8+
p bikes_cat.save
9+
p bikes_cat.errors
10+
11+
exit
12+
13+
p Craigslist::Category.all
14+
515
my_bike = Craigslist::Posting.new(
6-
category: 'bikes',
16+
category: bikes_cat,
717
title: 'My Mountain Bike for Sale',
818
price: 350.00,
919
make: 'Santa Cruz'
1020
)
1121

1222
# Test Craigslist::Posting attributes
1323

14-
p my_bike.category == 'bikes'
24+
p my_bike.category == bikes_cat
1525
p my_bike.title == 'My Mountain Bike for Sale'
1626
p my_bike.price == 350.00
1727
p my_bike.make == 'Santa Cruz'
1828

1929
# Test Craigslist::Posting persistence
2030

21-
p Craigslist.all.count == 0
31+
p Craigslist::Posting.count == 0
2232
my_bike.save
23-
p Craigslist.all.count == 1
33+
p Craigslist::Posting.count == 1
2434

2535
# Test Craigslist.all returns all postings
2636

27-
bike_two = Craigslist::Posting.new(category: 'bikes', title: 'Carbon Fibre Racing Bike', price: 500.00, make: 'Raleigh')
37+
bike_two = Craigslist::Posting.new(category: bikes_cat, title: 'Carbon Fibre Racing Bike', price: 500.00, make: 'Raleigh')
2838
bike_two.save
2939

30-
p Craigslist.all.count == 2
31-
p Craigslist.all.all? { |item| item.class == Craigslist::Posting }
40+
p Craigslist::Posting.all.count == 2
41+
p Craigslist::Posting.all.all? { |item| item.class == Craigslist::Posting }
3242

3343
# Search for a bike by make and price
34-
car = Craigslist::Posting.new(category: 'cars', title: 'Volkswagen Rabbit', price: 1000.00, make: 'Volkswagen')
44+
car_cat = Craigslist::Category.new(title: "cars", url_keystring: 'car')
45+
car_cat.save
46+
47+
car = Craigslist::Posting.new(category: car_cat, title: 'Volkswagen Rabbit', price: 1000.00, make: 'Volkswagen')
3548
car.save
3649

37-
bikes = Craigslist.bikes(make: 'Santa Cruz', max_price: 400.00)
50+
p bikes = Craigslist.bikes#(make: 'Santa Cruz', max_price: 400.00)
3851
# => return an array of Craigslist::Posting objects
3952

4053
bikes.first.title == 'My Mountain Bike for Sale'

0 commit comments

Comments
 (0)