30 lines
626 B
Go
30 lines
626 B
Go
|
package txt
|
||
|
|
|
||
|
|
import (
|
||
|
|
"unicode"
|
||
|
|
)
|
||
|
|
|
||
|
|
// isSeparator reports whether the rune could mark a word boundary.
|
||
|
|
func isSeparator(r rune) bool {
|
||
|
|
// ASCII alphanumerics and underscore are not separators
|
||
|
|
if r <= 0x7F {
|
||
|
|
switch {
|
||
|
|
case '0' <= r && r <= '9':
|
||
|
|
return false
|
||
|
|
case 'a' <= r && r <= 'z':
|
||
|
|
return false
|
||
|
|
case 'A' <= r && r <= 'Z':
|
||
|
|
return false
|
||
|
|
case r == '_', r == '\'':
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
// Letters and digits are not separators
|
||
|
|
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
// Otherwise, all we can do for now is treat spaces as separators.
|
||
|
|
return unicode.IsSpace(r)
|
||
|
|
}
|