Skip to content

Commit 13c584d

Browse files
committed
add script to generate db
1 parent ec42391 commit 13c584d

File tree

4 files changed

+400
-47
lines changed

4 files changed

+400
-47
lines changed

ebird/cmd/ebird_import.go

Lines changed: 53 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -17,52 +17,52 @@ import (
1717
)
1818

1919
type Obs struct {
20-
GlobalUniqueIdentifier string
21-
LastEditedDate string
22-
TaxonomicOrder string
23-
Category string
24-
CommonName string
25-
ScientificName string
26-
SubspeciesCommonName string
27-
SubspeciesScientificName string
28-
ObservationCount int
29-
BreedingBirdAtlasCode string
30-
BreedingBirdAtlasCategory string
31-
AgeSex string
32-
Country string
33-
CountryCode string
34-
State string
35-
StateCode string
36-
County string
37-
CountyCode string
38-
IBACode string
39-
BCRCode string
40-
USFWSCode string
41-
AtlasBlock string
42-
Locality string
43-
LocalityID string
44-
LocalityType string
45-
Latitude float64
46-
Longitude float64
47-
ObservationDate string
48-
TimeObservationsStarted string
49-
ObserverID string
50-
SamplingEventIdentifier string
51-
ProtocolType string
52-
ProtocolCode string
53-
ProjectCode string
54-
DurationMinutes int
55-
EffortDistanceKM float64
56-
EffortAreaHA float64
57-
NumberObservers int
58-
AllSpeciesReported bool
59-
GroupIdentifier string
60-
HasMedia bool
61-
Approved bool
62-
Reviewed bool
63-
Reason string
64-
TripComments string
65-
SpeciesComments string
20+
GlobalUniqueIdentifier string `json:"global_unique_identifier"`
21+
LastEditedDate string `json:"last_edited_date"`
22+
TaxonomicOrder string `json:"taxonomic_order"`
23+
Category string `json:"category"`
24+
CommonName string `json:"common_name"`
25+
ScientificName string `json:"scientific_name"`
26+
SubspeciesCommonName string `json:"subspecies_common_name"`
27+
SubspeciesScientificName string `json:"subspecies_scientific_name"`
28+
ObservationCount int `json:"observation_count"`
29+
BreedingBirdAtlasCode string `json:"breeding_bird_atlas_code"`
30+
BreedingBirdAtlasCategory string `json:"breeding_bird_atlas_category"`
31+
AgeSex string `json:"age_sex"`
32+
Country string `json:"country"`
33+
CountryCode string `json:"country_code"`
34+
State string `json:"state"`
35+
StateCode string `json:"state_code"`
36+
County string `json:"county"`
37+
CountyCode string `json:"county_code"`
38+
IBACode string `json:"iba_code"`
39+
BCRCode string `json:"bcr_code"`
40+
USFWSCode string `json:"usfws_code"`
41+
AtlasBlock string `json:"atlas_block"`
42+
Locality string `json:"locality"`
43+
LocalityID string `json:"locality_id"`
44+
LocalityType string `json:"locality_type"`
45+
Latitude float64 `json:"latitude"`
46+
Longitude float64 `json:"longitude"`
47+
ObservationDate string `json:"observation_date"`
48+
TimeObservationsStarted string `json:"time_observations_started"`
49+
ObserverID string `json:"observer_id"`
50+
SamplingEventIdentifier string `json:"sampling_event_identifier"`
51+
ProtocolType string `json:"protocol_type"`
52+
ProtocolCode string `json:"protocol_code"`
53+
ProjectCode string `json:"project_code"`
54+
DurationMinutes int `json:"duration_minutes"`
55+
EffortDistanceKM float64 `json:"effort_distance_km"`
56+
EffortAreaHA float64 `json:"effort_area_ha"`
57+
NumberObservers int `json:"number_observers"`
58+
AllSpeciesReported bool `json:"all_species_reported"`
59+
GroupIdentifier string `json:"group_identifier"`
60+
HasMedia bool `json:"has_media"`
61+
Approved bool `json:"approved"`
62+
Reviewed bool `json:"reviewed"`
63+
Reason string `json:"reason"`
64+
TripComments string `json:"trip_comments"`
65+
SpeciesComments string `json:"species_comments"`
6666
}
6767

6868
func decodeObs(vals []string) (Obs, error) {
@@ -222,6 +222,8 @@ func main() {
222222
var hasRow bool
223223
var values []string
224224
hasRow = scn.Scan()
225+
db.Exec(`BEGIN;`)
226+
i := 0
225227
for hasRow == true {
226228
hasRow = scn.Scan()
227229
if hasRow == false {
@@ -239,5 +241,10 @@ func main() {
239241
if err != nil {
240242
log.Fatal(err)
241243
}
244+
i++
245+
if i%1000 == 0 {
246+
fmt.Printf("\rprocessed %d records...", i)
247+
}
242248
}
249+
db.Exec(`COMMIT;`)
243250
}

ebird/server/server.go

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
package main
2+
3+
import (
4+
"database/sql"
5+
"encoding/json"
6+
"fmt"
7+
"log"
8+
"net/http"
9+
"os"
10+
"sort"
11+
"strconv"
12+
"text/tabwriter"
13+
"time"
14+
15+
"github.com/bsurc/pracprog/ebird"
16+
_ "github.com/mattn/go-sqlite3"
17+
)
18+
19+
// Sync with cmd/ebird_import.go
20+
type Obs struct {
21+
GlobalUniqueIdentifier string `json:"global_unique_identifier,omitempty"`
22+
LastEditedDate string `json:"last_edited_date,omitempty"`
23+
TaxonomicOrder string `json:"taxonomic_order,omitempty"`
24+
Category string `json:"category,omitempty"`
25+
CommonName string `json:"common_name,omitempty"`
26+
ScientificName string `json:"scientific_name,omitempty"`
27+
SubspeciesCommonName string `json:"subspecies_common_name,omitempty"`
28+
SubspeciesScientificName string `json:"subspecies_scientific_name,omitempty"`
29+
ObservationCount int `json:"observation_count,omitempty"`
30+
BreedingBirdAtlasCode string `json:"breeding_bird_atlas_code,omitempty"`
31+
BreedingBirdAtlasCategory string `json:"breeding_bird_atlas_category,omitempty"`
32+
AgeSex string `json:"age_sex,omitempty"`
33+
Country string `json:"country,omitempty"`
34+
CountryCode string `json:"country_code,omitempty"`
35+
State string `json:"state,omitempty"`
36+
StateCode string `json:"state_code,omitempty"`
37+
County string `json:"county,omitempty"`
38+
CountyCode string `json:"county_code,omitempty"`
39+
IBACode string `json:"iba_code,omitempty"`
40+
BCRCode string `json:"bcr_code,omitempty"`
41+
USFWSCode string `json:"usfws_code,omitempty"`
42+
AtlasBlock string `json:"atlas_block,omitempty"`
43+
Locality string `json:"locality"`
44+
LocalityID string `json:"locality_id,omitempty"`
45+
LocalityType string `json:"locality_type,omitempty"`
46+
Latitude float64 `json:"latitude,omitempty"`
47+
Longitude float64 `json:"longitude,omitempty"`
48+
ObservationDate string `json:"observation_date,omitempty"`
49+
TimeObservationsStarted string `json:"time_observations_started,omitempty"`
50+
ObserverID string `json:"observer_id,omitempty"`
51+
SamplingEventIdentifier string `json:"sampling_event_identifier,omitempty"`
52+
ProtocolType string `json:"protocol_type,omitempty"`
53+
ProtocolCode string `json:"protocol_code,omitempty"`
54+
ProjectCode string `json:"project_code,omitempty"`
55+
DurationMinutes int `json:"duration_minutes,omitempty"`
56+
EffortDistanceKM float64 `json:"effort_distance_km,omitempty"`
57+
EffortAreaHA float64 `json:"effort_area_ha,omitempty"`
58+
NumberObservers int `json:"number_observers,omitempty"`
59+
AllSpeciesReported bool `json:"all_species_reported,omitempty"`
60+
GroupIdentifier string `json:"group_identifier,omitempty"`
61+
HasMedia bool `json:"has_media,omitempty"`
62+
Approved bool `json:"approved,omitempty"`
63+
Reviewed bool `json:"reviewed,omitempty"`
64+
Reason string `json:"reason,omitempty"`
65+
TripComments string `json:"trip_comments"`
66+
SpeciesComments string `json:"species_comments"`
67+
}
68+
69+
func obsHandler(w http.ResponseWriter, r *http.Request) {
70+
row := db.QueryRow(`
71+
SELECT common_name, age_sex, observation_count, locality, longitude, latitude,
72+
observation_date, species_comments
73+
FROM ebird
74+
WHERE county_code='US-ID-037' AND species_comments!=''
75+
ORDER BY RANDOM() LIMIT 1`)
76+
var obs Obs
77+
var count string
78+
err := row.Scan(&obs.CommonName, &obs.AgeSex, &count, &obs.Locality, &obs.Longitude,
79+
&obs.Latitude, &obs.ObservationDate, &obs.SpeciesComments)
80+
if err != nil {
81+
http.Error(w, err.Error(), http.StatusInternalServerError)
82+
return
83+
}
84+
if count == "X" || count == "" {
85+
obs.ObservationCount = -1
86+
} else {
87+
obs.ObservationCount, _ = strconv.Atoi(count)
88+
}
89+
w.Header().Set("Content-Type", "application/json")
90+
err = json.NewEncoder(w).Encode(obs)
91+
if err != nil {
92+
http.Error(w, err.Error(), http.StatusInternalServerError)
93+
}
94+
}
95+
96+
func speciesHandler(w http.ResponseWriter, r *http.Request) {
97+
spp := r.FormValue("spp")
98+
if spp == "" {
99+
http.Error(w, "request URI must have a spp query parameter", http.StatusBadRequest)
100+
return
101+
}
102+
t, ok := taxa[spp]
103+
if !ok {
104+
http.Error(w, fmt.Sprintf("invalid spp: %s", spp), http.StatusBadRequest)
105+
return
106+
}
107+
108+
start := r.FormValue("start")
109+
_, err := time.Parse("2006-01-02", start)
110+
if err != nil || start == "" {
111+
http.Error(w, fmt.Sprintf("invalid start date: '%s'", start), http.StatusBadRequest)
112+
return
113+
}
114+
end := r.FormValue("end")
115+
_, err = time.Parse("2006-01-02", end)
116+
if err != nil || end == "" {
117+
http.Error(w, fmt.Sprintf("invalid end date: '%s'", end), http.StatusBadRequest)
118+
return
119+
}
120+
121+
rows, err := db.Query(`
122+
SELECT common_name, age_sex, observation_count, locality, longitude, latitude,
123+
observation_date, time_observations_started, species_comments
124+
FROM ebird
125+
WHERE common_name=? AND observation_date>? AND observation_date<?
126+
GROUP BY group_identifier
127+
ORDER BY observation_date`, t.PrimaryCommonName, start, end)
128+
if err != nil {
129+
http.Error(w, err.Error(), http.StatusInternalServerError)
130+
return
131+
}
132+
var (
133+
obs Obs
134+
obss []Obs
135+
count string
136+
)
137+
for rows.Next() {
138+
err := rows.Scan(&obs.CommonName, &obs.AgeSex, &count, &obs.Locality,
139+
&obs.Longitude, &obs.Latitude, &obs.ObservationDate,
140+
&obs.TimeObservationsStarted, &obs.SpeciesComments)
141+
if err != nil {
142+
http.Error(w, err.Error(), http.StatusInternalServerError)
143+
return
144+
}
145+
if count == "X" || count == "" {
146+
obs.ObservationCount = -1
147+
} else {
148+
obs.ObservationCount, _ = strconv.Atoi(count)
149+
}
150+
obss = append(obss, obs)
151+
}
152+
w.Header().Set("Content-Type", "application/json")
153+
err = json.NewEncoder(w).Encode(obss)
154+
if err != nil {
155+
http.Error(w, err.Error(), http.StatusInternalServerError)
156+
}
157+
}
158+
159+
func codeHandler(w http.ResponseWriter, r *http.Request) {
160+
tw := &tabwriter.Writer{}
161+
tw.Init(w, 0, 8, 0, '\t', 0)
162+
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", "category", "code", "common", "sci")
163+
for _, k := range taxaKeys {
164+
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n",
165+
taxa[k].Category,
166+
taxa[k].SpeciesCode,
167+
taxa[k].PrimaryCommonName,
168+
taxa[k].SciName,
169+
)
170+
}
171+
tw.Flush()
172+
/*
173+
TaxonOrder int
174+
Category string
175+
SpeciesCode string
176+
PrimaryCommonName string
177+
SciName string
178+
Order string
179+
Family string
180+
SpeciesGroup string
181+
ReportAs string
182+
*/
183+
}
184+
185+
var (
186+
db *sql.DB
187+
taxa map[string]ebird.Taxa
188+
taxaKeys []string
189+
)
190+
191+
const addr = ":8888"
192+
193+
func main() {
194+
var err error
195+
db, err = sql.Open("sqlite3", os.Args[1])
196+
if err != nil {
197+
log.Fatal(err)
198+
}
199+
200+
fin, err := os.Open("../taxa.csv")
201+
if err != nil {
202+
log.Fatal(err)
203+
}
204+
taxa, err = ebird.LoadTaxa(fin)
205+
if err != nil {
206+
log.Fatal(err)
207+
}
208+
for k := range taxa {
209+
taxaKeys = append(taxaKeys, k)
210+
}
211+
sort.Strings(taxaKeys)
212+
213+
mux := &http.ServeMux{}
214+
mux.HandleFunc("/", obsHandler)
215+
mux.HandleFunc("/species", speciesHandler)
216+
mux.HandleFunc("/codes", codeHandler)
217+
go func() {
218+
fmt.Println("open your browser to http://127.0.0.1" + addr)
219+
}()
220+
log.Fatal(http.ListenAndServe(addr, mux))
221+
}

0 commit comments

Comments
 (0)