Skip to content

Commit f545fb2

Browse files
ridiculousfishridiculousfish
committed
Work towards refactoring tokenizer to be a real object
1 parent e73be48 commit f545fb2

12 files changed

+122
-168
lines changed

‎builtin_commandline.cpp

+2-4
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,6 @@ static void write_part(const wchar_t *begin,
143143
int cut_at_cursor,
144144
int tokenize)
145145
{
146-
tokenizer tok;
147146
wcstring out;
148147
wchar_t *buff;
149148
size_t pos;
@@ -155,9 +154,8 @@ static void write_part(const wchar_t *begin,
155154
buff = wcsndup(begin, end-begin);
156155
// fwprintf( stderr, L"Subshell: %ls, end char %lc\n", buff, *end );
157156
out.clear();
158-
159-
for (tok_init(&tok, buff, TOK_ACCEPT_UNFINISHED);
160-
tok_has_next(&tok);
157+
tokenizer_t tok(buff, TOK_ACCEPT_UNFINISHED);
158+
for (; tok_has_next(&tok);
161159
tok_next(&tok))
162160
{
163161
if ((cut_at_cursor) &&

‎complete.cpp

+2-4
Original file line numberDiff line numberDiff line change
@@ -1765,7 +1765,6 @@ void complete(const wcstring &cmd, std::vector<completion_t> &comps, complete_ty
17651765
completer_t completer(cmd, type);
17661766

17671767
const wchar_t *tok_begin, *tok_end, *cmdsubst_begin, *cmdsubst_end, *prev_begin, *prev_end;
1768-
tokenizer tok;
17691768
const wchar_t *current_token=0, *prev_token=0;
17701769
wcstring current_command;
17711770
int on_command=0;
@@ -1807,9 +1806,8 @@ void complete(const wcstring &cmd, std::vector<completion_t> &comps, complete_ty
18071806

18081807
int had_cmd=0;
18091808
int end_loop=0;
1810-
1811-
tok_init(&tok, buff.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
1812-
1809+
1810+
tokenizer_t tok(buff.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
18131811
while (tok_has_next(&tok) && !end_loop)
18141812
{
18151813

‎fish_indent.cpp

+1-3
Original file line numberDiff line numberDiff line change
@@ -84,16 +84,14 @@ static void insert_tabs(wcstring &out, int indent)
8484
*/
8585
static int indent(wcstring &out, const wcstring &in, int flags)
8686
{
87-
tokenizer tok;
8887
int res=0;
8988
int is_command = 1;
9089
int indent = 0;
9190
int do_indent = 1;
9291
int prev_type = 0;
9392
int prev_prev_type = 0;
9493

95-
tok_init(&tok, in.c_str(), TOK_SHOW_COMMENTS);
96-
94+
tokenizer_t tok(in.c_str(), TOK_SHOW_COMMENTS);
9795
for (; tok_has_next(&tok); tok_next(&tok))
9896
{
9997
int type = tok_last_type(&tok);

‎fish_tests.cpp

+5-8
Original file line numberDiff line numberDiff line change
@@ -289,13 +289,12 @@ static void test_convert()
289289
*/
290290
static void test_tok()
291291
{
292-
tokenizer t;
293292

294293
say(L"Testing tokenizer");
295294

296295

297296
say(L"Testing invalid input");
298-
tok_init(&t, 0, 0);
297+
tokenizer_t t(NULL, 0);
299298

300299
if (tok_last_type(&t) != TOK_ERROR)
301300
{
@@ -326,14 +325,12 @@ static void test_tok()
326325
const int types[] =
327326
{
328327
TOK_STRING, TOK_REDIRECT_IN, TOK_STRING, TOK_REDIRECT_FD, TOK_STRING, TOK_STRING, TOK_STRING, TOK_REDIRECT_OUT, TOK_REDIRECT_APPEND, TOK_STRING, TOK_END
329-
}
330-
;
331-
size_t i;
328+
};
332329

333330
say(L"Test correct tokenization");
334-
335-
for (i=0, tok_init(&t, str, 0); i<(sizeof(types)/sizeof(int)); i++,tok_next(&t))
336-
{
331+
332+
tokenizer_t t(str, 0);
333+
for (size_t i=0; i < sizeof types / sizeof *types; i++, tok_next(&t)) {
337334
if (types[i] != tok_last_type(&t))
338335
{
339336
err(L"Tokenization error:");

‎highlight.cpp

+4-6
Original file line numberDiff line numberDiff line change
@@ -691,8 +691,8 @@ static bool autosuggest_parse_command(const wcstring &str, wcstring *out_command
691691
int arg_pos = -1;
692692

693693
bool had_cmd = false;
694-
tokenizer tok;
695-
for (tok_init(&tok, str.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS); tok_has_next(&tok); tok_next(&tok))
694+
tokenizer_t tok(str.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
695+
for (; tok_has_next(&tok); tok_next(&tok))
696696
{
697697
int last_type = tok_last_type(&tok);
698698

@@ -955,10 +955,8 @@ static void tokenize(const wchar_t * const buff, std::vector<int> &color, const
955955

956956
std::fill(color.begin(), color.end(), -1);
957957

958-
tokenizer tok;
959-
for (tok_init(&tok, buff, TOK_SHOW_COMMENTS | TOK_SQUASH_ERRORS);
960-
tok_has_next(&tok);
961-
tok_next(&tok))
958+
tokenizer_t tok(buff, TOK_SHOW_COMMENTS | TOK_SQUASH_ERRORS);
959+
for (; tok_has_next(&tok); tok_next(&tok))
962960
{
963961
int last_type = tok_last_type(&tok);
964962

‎history.cpp

+2-4
Original file line numberDiff line numberDiff line change
@@ -1415,10 +1415,8 @@ void history_t::add_with_file_detection(const wcstring &str)
14151415
ASSERT_IS_MAIN_THREAD();
14161416
path_list_t potential_paths;
14171417

1418-
tokenizer tokenizer;
1419-
for (tok_init(&tokenizer, str.c_str(), TOK_SQUASH_ERRORS);
1420-
tok_has_next(&tokenizer);
1421-
tok_next(&tokenizer))
1418+
tokenizer_t tokenizer(str.c_str(), TOK_SQUASH_ERRORS);
1419+
for (; tok_has_next(&tokenizer); tok_next(&tokenizer))
14221420
{
14231421
int type = tok_last_type(&tokenizer);
14241422
if (type == TOK_STRING)

‎parse_util.cpp

+5-13
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,6 @@ static void job_or_process_extent(const wchar_t *buff,
326326
wchar_t *buffcpy;
327327
int finished=0;
328328

329-
tokenizer tok;
330-
331329
CHECK(buff,);
332330

333331
if (a)
@@ -365,9 +363,8 @@ static void job_or_process_extent(const wchar_t *buff,
365363
DIE_MEM();
366364
}
367365

368-
for (tok_init(&tok, buffcpy, TOK_ACCEPT_UNFINISHED);
369-
tok_has_next(&tok) && !finished;
370-
tok_next(&tok))
366+
tokenizer_t tok(buffcpy, TOK_ACCEPT_UNFINISHED);
367+
for (; tok_has_next(&tok) && !finished; tok_next(&tok))
371368
{
372369
int tok_begin = tok_get_pos(&tok);
373370

@@ -440,8 +437,6 @@ void parse_util_token_extent(const wchar_t *buff,
440437
long pos;
441438
wchar_t *buffcpy;
442439

443-
tokenizer tok;
444-
445440
const wchar_t *a = NULL, *b = NULL, *pa = NULL, *pb = NULL;
446441

447442
CHECK(buff,);
@@ -474,9 +469,8 @@ void parse_util_token_extent(const wchar_t *buff,
474469
DIE_MEM();
475470
}
476471

477-
for (tok_init(&tok, buffcpy, TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
478-
tok_has_next(&tok);
479-
tok_next(&tok))
472+
tokenizer_t tok(buffcpy, TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
473+
for (; tok_has_next(&tok); tok_next(&tok))
480474
{
481475
size_t tok_begin = tok_get_pos(&tok);
482476
size_t tok_end = tok_begin;
@@ -711,9 +705,7 @@ void parse_util_get_parameter_info(const wcstring &cmd, const size_t pos, wchar_
711705
wchar_t last_quote = '\0';
712706
int unfinished;
713707

714-
tokenizer tok;
715-
tok_init(&tok, cmd.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
716-
708+
tokenizer_t tok(cmd.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
717709
for (; tok_has_next(&tok); tok_next(&tok))
718710
{
719711
if (tok_get_pos(&tok) > pos)

‎parser.cpp

+22-32
Original file line numberDiff line numberDiff line change
@@ -525,17 +525,15 @@ static int parser_is_pipe_forbidden(const wcstring &word)
525525
*/
526526
static const wchar_t *parser_find_end(const wchar_t * buff)
527527
{
528-
tokenizer tok;
529528
int had_cmd=0;
530529
int count = 0;
531530
int error=0;
532531
int mark=0;
533532

534533
CHECK(buff, 0);
535534

536-
for (tok_init(&tok, buff, 0);
537-
tok_has_next(&tok) && !error;
538-
tok_next(&tok))
535+
tokenizer_t tok(buff, 0);
536+
for (; tok_has_next(&tok) && !error; tok_next(&tok))
539537
{
540538
int last_type = tok_last_type(&tok);
541539
switch (last_type)
@@ -796,7 +794,6 @@ void parser_t::print_errors_stderr()
796794

797795
int parser_t::eval_args(const wchar_t *line, std::vector<completion_t> &args)
798796
{
799-
tokenizer tok;
800797

801798
expand_flags_t eflags = 0;
802799
if (! show_errors)
@@ -808,8 +805,8 @@ int parser_t::eval_args(const wchar_t *line, std::vector<completion_t> &args)
808805
eval_args may be called while evaulating another command, so we
809806
save the previous tokenizer and restore it on exit
810807
*/
811-
tokenizer *previous_tokenizer=current_tokenizer;
812-
int previous_pos=current_tokenizer_pos;
808+
tokenizer_t * const previous_tokenizer = current_tokenizer;
809+
const int previous_pos = current_tokenizer_pos;
813810
int do_loop=1;
814811

815812
CHECK(line, 1);
@@ -819,10 +816,10 @@ int parser_t::eval_args(const wchar_t *line, std::vector<completion_t> &args)
819816
if (this->parser_type == PARSER_TYPE_GENERAL)
820817
proc_push_interactive(0);
821818

819+
tokenizer_t tok(line, (show_errors ? 0 : TOK_SQUASH_ERRORS));
822820
current_tokenizer = &tok;
823821
current_tokenizer_pos = 0;
824822

825-
tok_init(&tok, line, (show_errors ? 0 : TOK_SQUASH_ERRORS));
826823
error_code=0;
827824

828825
for (; do_loop && tok_has_next(&tok) ; tok_next(&tok))
@@ -1319,7 +1316,7 @@ job_t *parser_t::job_get_from_pid(int pid)
13191316
*/
13201317
void parser_t::parse_job_argument_list(process_t *p,
13211318
job_t *j,
1322-
tokenizer *tok,
1319+
tokenizer_t *tok,
13231320
std::vector<completion_t> &args,
13241321
bool unskip)
13251322
{
@@ -1718,7 +1715,7 @@ f
17181715
*/
17191716
int parser_t::parse_job(process_t *p,
17201717
job_t *j,
1721-
tokenizer *tok)
1718+
tokenizer_t *tok)
17221719
{
17231720
std::vector<completion_t> args; // The list that will become the argc array for the program
17241721
int use_function = 1; // May functions be considered when checking what action this command represents
@@ -2185,7 +2182,6 @@ int parser_t::parse_job(process_t *p,
21852182

21862183
const wchar_t *end=parser_find_end(tok_string(tok) +
21872184
current_tokenizer_pos);
2188-
tokenizer subtok;
21892185
int make_sub_block = j->first_process != p;
21902186

21912187
if (!end)
@@ -2202,9 +2198,8 @@ int parser_t::parse_job(process_t *p,
22022198
{
22032199
int done=0;
22042200

2205-
for (tok_init(&subtok, end, 0);
2206-
!done && tok_has_next(&subtok);
2207-
tok_next(&subtok))
2201+
tokenizer_t subtok(end, 0);
2202+
for (; ! done && tok_has_next(&subtok); tok_next(&subtok))
22082203
{
22092204

22102205
switch (tok_last_type(&subtok))
@@ -2388,7 +2383,7 @@ static bool job_should_skip_elseif(const job_t *job, const block_t *current_bloc
23882383
\param tok The tokenizer to read tokens from
23892384
*/
23902385

2391-
void parser_t::eval_job(tokenizer *tok)
2386+
void parser_t::eval_job(tokenizer_t *tok)
23922387
{
23932388
ASSERT_IS_MAIN_THREAD();
23942389
job_t *j;
@@ -2630,7 +2625,7 @@ int parser_t::eval(const wcstring &cmdStr, const io_chain_t &io, enum block_type
26302625
const wchar_t * const cmd = cmdStr.c_str();
26312626
size_t forbid_count;
26322627
int code;
2633-
tokenizer *previous_tokenizer=current_tokenizer;
2628+
tokenizer_t *previous_tokenizer=current_tokenizer;
26342629
block_t *start_current_block = current_block;
26352630

26362631
/* Record the current chain so we can put it back later */
@@ -2676,8 +2671,7 @@ int parser_t::eval(const wcstring &cmdStr, const io_chain_t &io, enum block_type
26762671

26772672
this->push_block(new scope_block_t(block_type));
26782673

2679-
current_tokenizer = new tokenizer;
2680-
tok_init(current_tokenizer, cmd, 0);
2674+
current_tokenizer = new tokenizer_t(cmd, 0);
26812675

26822676
error_code = 0;
26832677

@@ -2907,19 +2901,17 @@ int parser_t::parser_test_argument(const wchar_t *arg, wcstring *out, const wcha
29072901

29082902
int parser_t::test_args(const wchar_t * buff, wcstring *out, const wchar_t *prefix)
29092903
{
2910-
tokenizer tok;
2911-
tokenizer *previous_tokenizer = current_tokenizer;
2912-
int previous_pos = current_tokenizer_pos;
2904+
tokenizer_t *const previous_tokenizer = current_tokenizer;
2905+
const int previous_pos = current_tokenizer_pos;
29132906
int do_loop = 1;
29142907
int err = 0;
29152908

29162909
CHECK(buff, 1);
29172910

2918-
current_tokenizer = &tok;
29192911

2920-
for (tok_init(&tok, buff, 0);
2921-
do_loop && tok_has_next(&tok);
2922-
tok_next(&tok))
2912+
tokenizer_t tok(buff, 0);
2913+
current_tokenizer = &tok;
2914+
for (; do_loop && tok_has_next(&tok); tok_next(&tok))
29232915
{
29242916
current_tokenizer_pos = tok_get_pos(&tok);
29252917
switch (tok_last_type(&tok))
@@ -2970,7 +2962,7 @@ int parser_t::test_args(const wchar_t * buff, wcstring *out, const wchar_t *pre
29702962

29712963
tok_destroy(&tok);
29722964

2973-
current_tokenizer=previous_tokenizer;
2965+
current_tokenizer = previous_tokenizer;
29742966
current_tokenizer_pos = previous_pos;
29752967

29762968
error_code=0;
@@ -2985,7 +2977,6 @@ int parser_t::test(const wchar_t * buff,
29852977
{
29862978
ASSERT_IS_MAIN_THREAD();
29872979

2988-
tokenizer tok;
29892980
/*
29902981
Set to one if a command name has been given for the currently
29912982
parsed process specification
@@ -2994,8 +2985,8 @@ int parser_t::test(const wchar_t * buff,
29942985
int err=0;
29952986
int unfinished = 0;
29962987

2997-
tokenizer *previous_tokenizer=current_tokenizer;
2998-
int previous_pos=current_tokenizer_pos;
2988+
tokenizer_t * const previous_tokenizer=current_tokenizer;
2989+
const int previous_pos=current_tokenizer_pos;
29992990

30002991
int block_pos[BLOCK_MAX_COUNT] = {};
30012992
block_type_t block_type[BLOCK_MAX_COUNT] = {};
@@ -3043,11 +3034,10 @@ int parser_t::test(const wchar_t * buff,
30433034

30443035
}
30453036

3037+
tokenizer_t tok(buff, 0);
30463038
current_tokenizer = &tok;
30473039

3048-
for (tok_init(&tok, buff, 0);
3049-
;
3050-
tok_next(&tok))
3040+
for (;; tok_next(&tok))
30513041
{
30523042
current_tokenizer_pos = tok_get_pos(&tok);
30533043

‎parser.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ struct profile_item_t
295295
wcstring cmd;
296296
};
297297

298-
struct tokenizer;
298+
struct tokenizer_t;
299299

300300
class parser_t
301301
{
@@ -316,7 +316,7 @@ class parser_t
316316
wcstring err_buff;
317317

318318
/** Pointer to the current tokenizer */
319-
tokenizer *current_tokenizer;
319+
tokenizer_t *current_tokenizer;
320320

321321
/** String for representing the current line */
322322
wcstring lineinfo;
@@ -344,10 +344,10 @@ class parser_t
344344
parser_t(const parser_t&);
345345
parser_t& operator=(const parser_t&);
346346

347-
void parse_job_argument_list(process_t *p, job_t *j, tokenizer *tok, std::vector<completion_t>&, bool);
348-
int parse_job(process_t *p, job_t *j, tokenizer *tok);
347+
void parse_job_argument_list(process_t *p, job_t *j, tokenizer_t *tok, std::vector<completion_t>&, bool);
348+
int parse_job(process_t *p, job_t *j, tokenizer_t *tok);
349349
void skipped_exec(job_t * j);
350-
void eval_job(tokenizer *tok);
350+
void eval_job(tokenizer_t *tok);
351351
int parser_test_argument(const wchar_t *arg, wcstring *out, const wchar_t *prefix, int offset);
352352
void print_errors(wcstring &target, const wchar_t *prefix);
353353
void print_errors_stderr();

0 commit comments

Comments
 (0)