@@ -637,6 +637,41 @@ impl Tokens {
637637 }
638638 }
639639
640+ /// Returns a slice of tokens before the given [`TextSize`] offset.
641+ ///
642+ /// If the given offset is between two tokens, the returned slice will end just before the
643+ /// following token. In other words, if the offset is between the end of previous token and
644+ /// start of next token, the returned slice will end just before the next token.
645+ ///
646+ /// # Panics
647+ ///
648+ /// If the given offset is inside a token range at any point
649+ /// other than the start of the range.
650+ pub fn before ( & self , offset : TextSize ) -> & [ Token ] {
651+ match self . binary_search_by ( |token| token. start ( ) . cmp ( & offset) ) {
652+ Ok ( idx) => & self [ ..idx] ,
653+ Err ( idx) => {
654+ // We can't use `saturating_sub` here because a file could contain a BOM header, in
655+ // which case the token starts at offset 3 for UTF-8 encoded file content.
656+ if idx > 0 {
657+ if let Some ( prev) = self . get ( idx - 1 ) {
658+ // If it's equal to the end offset, then it's at a token boundary which is
659+ // valid. If it's greater than the end offset, then it's in the gap between
660+ // the tokens which is valid as well.
661+ assert ! (
662+ offset >= prev. end( ) ,
663+ "Offset {:?} is inside a token range {:?}" ,
664+ offset,
665+ prev. range( )
666+ ) ;
667+ }
668+ }
669+
670+ & self [ ..idx]
671+ }
672+ }
673+ }
674+
640675 /// Returns a slice of tokens after the given [`TextSize`] offset.
641676///
642677/// If the given offset is between two tokens, the returned slice will start from the following
@@ -645,7 +680,8 @@ impl Tokens {
645680///
646681/// # Panics
647682///
648- /// If the given offset is inside a token range.
683+ /// If the given offset is inside a token range at any point
684+ /// other than the start of the range.
649685pub fn after ( & self , offset : TextSize ) -> & [ Token ] {
650686 match self . binary_search_by ( |token| token. start ( ) . cmp ( & offset) ) {
651687 Ok ( idx) => & self [ idx..] ,
@@ -947,6 +983,68 @@ mod tests {
947983 tokens. after ( TextSize :: new ( 5 ) ) ;
948984 }
949985
986+ #[ test]
987+ fn tokens_before_offset_at_first_token_start ( ) {
988+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
989+ let before = tokens. before ( TextSize :: new ( 0 ) ) ;
990+ assert_eq ! ( before. len( ) , 0 ) ;
991+ }
992+
993+ #[ test]
994+ fn tokens_before_offset_after_first_token_gap ( ) {
995+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
996+ let before = tokens. before ( TextSize :: new ( 3 ) ) ;
997+ assert_eq ! ( before. len( ) , 1 ) ;
998+ assert_eq ! ( before. last( ) . unwrap( ) . kind( ) , TokenKind :: Def ) ;
999+ }
1000+
1001+ #[ test]
1002+ fn tokens_before_offset_at_second_token_start ( ) {
1003+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
1004+ let before = tokens. before ( TextSize :: new ( 4 ) ) ;
1005+ assert_eq ! ( before. len( ) , 1 ) ;
1006+ assert_eq ! ( before. last( ) . unwrap( ) . kind( ) , TokenKind :: Def ) ;
1007+ }
1008+
1009+ #[ test]
1010+ fn tokens_before_offset_at_token_start ( ) {
1011+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
1012+ let before = tokens. before ( TextSize :: new ( 8 ) ) ;
1013+ assert_eq ! ( before. len( ) , 3 ) ;
1014+ assert_eq ! ( before. last( ) . unwrap( ) . kind( ) , TokenKind :: Lpar ) ;
1015+ }
1016+
1017+ #[ test]
1018+ fn tokens_before_offset_at_token_end ( ) {
1019+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
1020+ let before = tokens. before ( TextSize :: new ( 11 ) ) ;
1021+ assert_eq ! ( before. len( ) , 6 ) ;
1022+ assert_eq ! ( before. last( ) . unwrap( ) . kind( ) , TokenKind :: Newline ) ;
1023+ }
1024+
1025+ #[ test]
1026+ fn tokens_before_offset_between_tokens ( ) {
1027+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
1028+ let before = tokens. before ( TextSize :: new ( 13 ) ) ;
1029+ assert_eq ! ( before. len( ) , 6 ) ;
1030+ assert_eq ! ( before. last( ) . unwrap( ) . kind( ) , TokenKind :: Newline ) ;
1031+ }
1032+
1033+ #[ test]
1034+ fn tokens_before_offset_at_last_token_end ( ) {
1035+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
1036+ let before = tokens. before ( TextSize :: new ( 33 ) ) ;
1037+ assert_eq ! ( before. len( ) , 10 ) ;
1038+ assert_eq ! ( before. last( ) . unwrap( ) . kind( ) , TokenKind :: Pass ) ;
1039+ }
1040+
1041+ #[ test]
1042+ #[ should_panic( expected = "Offset 5 is inside a token range 4..7" ) ]
1043+ fn tokens_before_offset_inside_token ( ) {
1044+ let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
1045+ tokens. before ( TextSize :: new ( 5 ) ) ;
1046+ }
1047+
9501048 #[ test]
9511049 fn tokens_in_range_at_token_offset ( ) {
9521050 let tokens = new_tokens ( TEST_CASE_WITH_GAP . into_iter ( ) ) ;
0 commit comments