summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--syntax-highlight/lexers.scm39
1 files changed, 39 insertions, 0 deletions
diff --git a/syntax-highlight/lexers.scm b/syntax-highlight/lexers.scm
index 7ff5b09..0591da2 100644
--- a/syntax-highlight/lexers.scm
+++ b/syntax-highlight/lexers.scm
@@ -41,6 +41,7 @@
empty-tokens
token-add
+ token-append
token-drop
token-take
token-peek
@@ -58,6 +59,7 @@
lex-all
lex-zero-or-more
lex-consume
+ lex-consume-until
lex-maybe
lex-regexp
lex-string
@@ -112,6 +114,9 @@ position."
(define (token-add tokens new-token)
(cons new-token tokens))
+(define (token-append tokens new-tokens)
+ (append new-tokens tokens))
+
(define (token-peek tokens)
(car tokens))
@@ -125,6 +130,13 @@ position."
;; Tokens are accumulated as a stack i.e. in reverse order.
(reverse tokens))
+(define (token-tag tokens tag)
+ (map
+ (match-lambda
+ ((? string? tok) (list tag tok))
+ ((? list? tok) (cons tag tok)))
+ tokens))
+
;;;
;;; Lexers
@@ -227,6 +239,33 @@ returned as plain strings."
consume)
+(define* (lex-consume-until until lexer #:key (tag #f))
+ "Return a lexer that always succeeds with a list of as many consecutive
+successful applications of LEXER as possible, consuming the entire input text,
+or until a successful match of UNTIL, whichever comes first. Sections of text
+that could not be lexed are returned as plain strings."
+ (define (substring* cursor start)
+ (substring (cursor-text cursor) start (cursor-position cursor)))
+
+ (define (consume-until tokens cursor)
+ (if (cursor-end? cursor)
+ (values tokens cursor)
+ (let-values (((result remainder) (until tokens cursor)))
+ (if result
+ (values result remainder)
+ (let-values (((result remainder) (lexer tokens cursor)))
+ (if result
+ (consume-until result remainder)
+ (values (token-add tokens (cursor->string cursor))
+ (move-cursor-to-end cursor))))))))
+
+ (if tag
+ (lambda (tokens cursor)
+ (let-values (((result remainder) (consume-until empty-tokens cursor)))
+ (let ((result (token-tag result tag)))
+ (values (token-append tokens result) remainder))))
+ consume-until))
+
(define (lex-maybe lexer)
"Create a lexer that always succeeds, but tries to use LEXER. If
LEXER succeeds, its result is returned, otherwise the empty string is