Skip to content

Commit

Permalink
bugfix
Browse files Browse the repository at this point in the history
  • Loading branch information
flozi00 authored Jul 5, 2020
1 parent ce53c30 commit 132ae7a
Showing 1 changed file with 8 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1881,13 +1881,12 @@ def __init__(self, data_file, tokenizer, args):

@staticmethod
def _get_n_lines(data_file):
with open(data_file, "r+", encoding="utf-8") as f:
buf = mmap.mmap(f.fileno(), 0)
lines = 0
readline = buf.readline
while readline():
lines += 1
return lines
counter = 0
myfile = open(data_file,"r+")
for line in myfile:
counter += 1

return counter

# @staticmethod
# def _get_n_lines(data_file):
Expand All @@ -1898,6 +1897,8 @@ def _get_n_lines(data_file):
# return line_idx

def __getitem__(self, idx):
if(idx == 0):
idx = 1
line = linecache.getline(self.data_file, idx)
qa_sample = json.loads(line)
example = get_examples([qa_sample])[0]
Expand Down

0 comments on commit 132ae7a

Please sign in to comment.