UNB/ CS/ David Bremner/ teaching/ cs3613/ code/ doge-lexer.rkt
#lang racket/base
(require parser-tools/lex (prefix-in : parser-tools/lex-sre))
(require ragg/support)
(provide tokenize tokenize-string)
(require rackunit)
(define (tokenize-string str)
  (let ([ip (open-input-string str)])
	(tokenize ip)))

(define (tokenize ip)
    (define my-lexer
      (lexer-src-pos
       [(:or "very" "wow" "such" "language" "prof" "mean" "difficult" "tricky" "course" ".")
	(token lexeme lexeme)]
       [whitespace
         (token 'WHITESPACE lexeme #:skip? #t)]
       [(eof)
	(void)]))
    (define (next-token) (my-lexer ip))
    next-token)

(define toks (tokenize (open-input-string #<<EOF
wow. such language. very tricky. wow.
wow. such course. very tricky. wow.
wow. such course. very difficult. wow.
wow. such prof. very mean. wow.
EOF
)))

(check-equal? 
 (build-list 16 (lambda (n) (token-struct-type (position-token-token (toks)))))
 '(wow |.| WHITESPACE such WHITESPACE language |.| WHITESPACE very WHITESPACE tricky |.| WHITESPACE wow |.| WHITESPACE))