Tokenizer
Implementation sample
tokenize.ts
type Token = {
type: 'Reserved' | 'Number' | 'EOF'
value: string
next: Token | null | undefined
}
function tokenize(input: string): Token {
let head = {
type: 'Reserved',
value: '',
next: null
}
let token = head
while (!atEOF(input)) {
let char = input[current]
if (char === '+') {
token.next = {
type: 'Reserved',
value: '+',
next: null
}
} else if (char === '-') {
token.next = {
type: 'Reserved',
value: '-',
next: null
}
} else if (/[0-9]/.test(char)) {
let value = ''
while (/[0-9]/.test(char)) {
value += char
char = input[++current]
}
token.next = {
type: 'Number',
value,
next: null
}
} else {
throw new Error('unexpected character')
}
token = token.next
}
token.next = {
type: 'EOF',
value: '',
next: null
}
return head.next