-
Notifications
You must be signed in to change notification settings - Fork 105
/
import.rb
92 lines (84 loc) · 2.71 KB
/
import.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
require 'bundler/setup'
require 'sequel'
require 'mysql2'
require 'bible_parser'
require 'bible_ref'
DB = Sequel.connect(ENV['DATABASE_URL'].sub(%r{mysql://}, 'mysql2://'), charset: 'utf8')
class Importer
def import(path, translation_id)
puts ' importing...'
bible = BibleParser.new(File.open(path))
bible.each_verse do |verse|
data = verse.to_h
data[:book] = data.delete(:book_title)
data[:chapter] = data.delete(:chapter_num)
data[:verse] = data.delete(:num)
data[:translation_id] = translation_id
print " #{translation_id} - #{data[:book]} #{data[:chapter]}:#{data[:verse]} \r"
DB[:verses].insert(data)
end
puts ' done '
end
end
if ARGV.include?('--drop-tables')
DB.drop_table :translations
DB.drop_table :verses
end
DB.create_table? :translations, charset: 'utf8' do
primary_key :id
String :identifier
String :name
String :language
String :language_code
String :license
end
DB.create_table? :verses, charset: 'utf8' do
primary_key :id
Fixnum :book_num
String :book_id
String :book
Fixnum :chapter
Fixnum :verse
String :text, text: true
Fixnum :translation_id
end
importer = Importer.new
# grab bible file info from the README.md table (markdown format)
table = File.read('bibles/README.md').scan(/^ *\|.+\| *$/)
headings = table.shift.split(/\s*\|\s*/)
table.shift # junk
translations = table.map do |row|
cells = row.split(/\s*\|\s*/)
headings.each_with_index.each_with_object({}) do |(heading, index), hash|
hash[heading.downcase] = cells[index] unless heading.empty?
end
end
translations.each do |translation|
path = "bibles/#{translation['filename']}"
puts path
lang_code_and_id = translation.delete('filename').split('.').first
lang_parts = lang_code_and_id.split('-')
if lang_parts.size == 3
translation['language_code'] = lang_parts.first
translation['identifier'] = translation['abbrev'].downcase
raise 'bad abbrev' if translation['identifier'].to_s.strip == ''
elsif lang_parts.size == 2
translation['language_code'], translation['identifier'] = lang_parts
else
raise "error with language and id for lang parts: #{lang_parts.inspect}"
end
translation.delete('format')
translation.delete('abbrev')
translation['name'] = translation.delete('version')
language = translation['language_code'].split('-').first
begin
BibleRef::Reference.new('John 3:16', language: language)
rescue KeyError
puts " language #{language} not supported"
next
end
unless DB['select id from translations where identifier = ?', translation['identifier']].any?
id = DB[:translations].insert(translation)
importer.import(path, id)
end
end