-
Notifications
You must be signed in to change notification settings - Fork 3
/
token.go
94 lines (78 loc) · 1.33 KB
/
token.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package lad
import (
"unicode"
)
type token struct {
index int
origin []rune
input []rune
}
func newToken(raw string) *token {
return &token{
input: []rune(raw),
origin: []rune(raw),
index: -1,
}
}
func (t *token) next() string {
for unicode.IsSpace(t.current()) {
t.moveNext()
}
current := t.current()
if current == -1 {
return ""
}
if !t.currentIsHan() {
return t.readWord()
}
t.moveNext()
return string(current)
}
func (t *token) readWord() string {
word := []rune{}
for {
if unicode.IsSpace(t.current()) ||
t.currentIsHan() || t.currentIs(-1) {
return string(word)
}
word = append(word, t.current())
t.moveNext()
}
}
func (t *token) currentIsHan() bool {
return unicode.Is(unicode.Han, t.current())
}
func (t *token) currentIs(runes ...rune) bool {
for _, r := range runes {
if r == t.current() {
return true
}
}
return false
}
func (t *token) moveNext() {
t.index++
t.input = t.input[1:]
}
func (t *token) prevNStr(index, preN int) string {
rs := make([]rune, 0)
for preN > 0 {
r := t.origin[index]
index--
rs = append([]rune{r}, rs...)
if unicode.IsSpace(r) {
continue
}
preN--
}
return string(rs)
}
func (t *token) current() rune {
return t.getRuneAt(0)
}
func (t *token) getRuneAt(i int) rune {
if len(t.input) == i {
return -1
}
return t.input[i]
}