UNB/ CS/ David Bremner/ teaching/ cs3613/ code/ dog-lexer.rkt
#lang racket/base
(require parser-tools/lex (prefix-in : parser-tools/lex-sre))
(require ragg/support)
(provide tokenize tokenize-string)
(require rackunit)
(define (tokenize-string str)
  (let ([ip (open-input-string str)])
	(tokenize ip)))

(define (tokenize ip)
    (define my-lexer
      (lexer-src-pos
       [(:or "bark" "growl" "slobber" "wag" "bite" ".")
	(token lexeme lexeme)]
       [whitespace
         (token 'WHITESPACE lexeme #:skip? #t)]
       [(eof)
	(void)]))
    (define (next-token) (my-lexer ip))
    next-token)

(define toks (tokenize (open-input-string #<<EOF
bark bark slobber wag.
wag wag wag bite
EOF
)))

(check-equal? 
 (build-list 16 (lambda (n) (token-struct-type (position-token-token (toks)))))
'(bark WHITESPACE bark WHITESPACE slobber WHITESPACE wag |.|
       WHITESPACE wag WHITESPACE wag WHITESPACE wag WHITESPACE bite))