Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 | 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 26x 3328x 3328x 1352x 3328x 260x 3328x 104x 3328x 624x 3328x 52x 3328x 26x 3328x 26x 3328x 26x 3328x 52x 3328x 52x 26x 104357x 26x 26x 44092x 44057x 35x 3x 32x 65876x 70x 38298x 31x 7x 15x 14x | /**
* Character classification flags — O(1) lookup via pre-computed bitmask table
*
* Parallels Spring Tokenizer character classification.
* Uint8Array pre-computed table for O(1) ASCII range queries.
*/
export enum CharFlag {
NONE = 0,
LETTER = 1 << 0, // a-z, A-Z
DIGIT = 1 << 1, // 0-9
WHITESPACE = 1 << 2, // space, \t, \r, \n
OPERATOR = 1 << 3, // + - * / % ^ < > = ! | & ? : . , ( ) [ ] { } @ #
QUOTE = 1 << 4, // ' "
UNDERSCORE = 1 << 5, // _
DOLLAR = 1 << 6, // $
DOT = 1 << 7, // .
EXPONENT = 1 << 8, // e, E
SIGN = 1 << 9, // +, - (for numeric context)
}
// Uint16Array: CharFlag range includes EXPONENT (256) and SIGN (512),
// which exceed Uint8Array max (255).
const CHAR_FLAG_TABLE = new Uint16Array(128);
const UNDERSCORE = 0x5f;
const DOLLAR = 0x24;
function buildCharTable(): void {
for (let c = 0; c < 128; c++) {
const ch = String.fromCharCode(c);
if ((c >= 65 && c <= 90) || (c >= 97 && c <= 122)) {
CHAR_FLAG_TABLE[c]! |= CharFlag.LETTER;
}
if (c >= 48 && c <= 57) {
CHAR_FLAG_TABLE[c]! |= CharFlag.DIGIT;
}
if (ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n') {
CHAR_FLAG_TABLE[c]! |= CharFlag.WHITESPACE;
}
if ('+-*/%^<>=!|&?:,.()[]{}@#'.includes(ch)) {
CHAR_FLAG_TABLE[c]! |= CharFlag.OPERATOR;
}
if (ch === "'" || ch === '"') {
CHAR_FLAG_TABLE[c]! |= CharFlag.QUOTE;
}
// '_' = 95
if (c === 95) {
CHAR_FLAG_TABLE[c]! |= CharFlag.UNDERSCORE;
}
// '$' = 36
if (c === 36) {
CHAR_FLAG_TABLE[c]! |= CharFlag.DOLLAR;
}
// '.' = 46
if (c === 46) {
CHAR_FLAG_TABLE[c]! |= CharFlag.DOT;
}
// 'E' = 69, 'e' = 101
if (c === 69 || c === 101) {
CHAR_FLAG_TABLE[c]! |= CharFlag.EXPONENT;
}
// '+' = 43, '-' = 45
if (c === 43 || c === 45) {
CHAR_FLAG_TABLE[c]! |= CharFlag.SIGN;
}
}
}
// Build character table at module initialization
buildCharTable();
/**
* Get character flag bitmask.
*
* The pre-computed table covers ASCII only; callers that need Unicode
* awareness (identifier letters) must use {@link isLetter} instead.
*/
export function getCharFlag(ch: number): number {
return ch >= 0 && ch < 128 ? CHAR_FLAG_TABLE[ch]! : CharFlag.NONE;
}
/**
* Unicode letter test used as the slow path for non-ASCII input.
*
* Spring's `Tokenizer#isAlphabetic` delegates to `Character.isLetter`, which
* accepts every Unicode letter — CJK ideographs, accented Latin, Greek and
* Cyrillic alike. Mirroring that is load-bearing rather than cosmetic: it is
* what makes identifiers such as `年龄` or `café` legal, and the Chinese
* natural-language pipeline emits exactly those.
*/
const UNICODE_LETTER = /\p{L}/u;
/** Highest UTF-16 code unit. The tokenizer never supplies a value above this. */
const MAX_CODE_UNIT = 0xffff;
/**
* Check whether a UTF-16 code unit is a letter in identifier position.
*
* ASCII is answered from the pre-computed table; everything else falls back to
* a Unicode letter test.
*
* The parameter is a code *unit*, not a code point, because the tokenizer
* advances with `charCodeAt`. Two consequences follow, and both match Spring's
* `char`-based tokenizer: a surrogate half of an astral-plane character is not
* a letter, and a value above the BMP is not a letter either.
*/
export function isLetter(ch: number): boolean {
if (ch >= 0 && ch < 128) {
return (CHAR_FLAG_TABLE[ch]! & CharFlag.LETTER) !== 0;
}
if (ch < 0 || ch > MAX_CODE_UNIT) {
return false;
}
return UNICODE_LETTER.test(String.fromCharCode(ch));
}
/**
* Check if character is a digit (0-9)
*/
export function isDigit(ch: number): boolean {
return (getCharFlag(ch) & CharFlag.DIGIT) !== 0;
}
/**
* Check if character is a hex digit (0-9, a-f, A-F)
*/
export function isHexDigit(ch: number): boolean {
return (
isDigit(ch) ||
(ch >= 65 && ch <= 70) || // A-F
(ch >= 97 && ch <= 102)
); // a-f
}
/**
* Check if character is whitespace
*/
export function isWhitespace(ch: number): boolean {
return (getCharFlag(ch) & CharFlag.WHITESPACE) !== 0;
}
/**
* Check if character is an operator
*/
export function isOperator(ch: number): boolean {
return (getCharFlag(ch) & CharFlag.OPERATOR) !== 0;
}
/**
* Check if character is a quote
*/
export function isQuote(ch: number): boolean {
return (getCharFlag(ch) & CharFlag.QUOTE) !== 0;
}
/**
* Check if character is a valid identifier start (letter, _, $)
*/
export function isIdentifierStart(ch: number): boolean {
return isLetter(ch) || ch === UNDERSCORE || ch === DOLLAR;
}
/**
* Check if character is a valid identifier part (letter, digit, _, $)
*
* Note that `isDigit` stays ASCII-only, matching Spring's `FLAGS` table which
* marks only `0`-`9`; a full-width digit is therefore not an identifier part.
*/
export function isIdentifierPart(ch: number): boolean {
return isLetter(ch) || isDigit(ch) || ch === UNDERSCORE || ch === DOLLAR;
}
|