You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
18 lines
326 B
18 lines
326 B
3 weeks ago
|
package model
|
||
|
|
|
||
|
|
const (
|
||
|
|
TOKEN_TYPE_NORMAL = iota + 1
|
||
|
|
TOKEN_TYPE_UNKNOWN
|
||
|
|
TOKEN_TYPE_CONTROL
|
||
|
|
TOKEN_TYPE_USER_DEFINED
|
||
|
|
TOKEN_TYPE_UNUSED
|
||
|
|
TOKEN_TYPE_BYTE
|
||
|
|
)
|
||
|
|
|
||
|
|
type TextProcessor interface {
|
||
|
|
Encode(s string, addSpecial bool) ([]int32, error)
|
||
|
|
Decode([]int32) (string, error)
|
||
|
|
Is(int32, Special) bool
|
||
|
|
Vocabulary() *Vocabulary
|
||
|
|
}
|