Files
n8n-openai-adapter/lib/elixir/src/elixir_tokenizer.hrl
T

46 lines
1.8 KiB
Erlang

%% SPDX-License-Identifier: Apache-2.0
%% SPDX-FileCopyrightText: 2021 The Elixir Team
%% Numbers
-define(is_hex(S), (?is_digit(S) orelse (S >= $A andalso S =< $F) orelse (S >= $a andalso S =< $f))).
-define(is_bin(S), (S >= $0 andalso S =< $1)).
-define(is_octal(S), (S >= $0 andalso S =< $7)).
%% Digits and letters
-define(is_digit(S), (S >= $0 andalso S =< $9)).
-define(is_upcase(S), (S >= $A andalso S =< $Z)).
-define(is_downcase(S), (S >= $a andalso S =< $z)).
%% Others
-define(is_quote(S), (S =:= $" orelse S =:= $')).
-define(is_sigil(S), (S =:= $/ orelse S =:= $< orelse S =:= $" orelse S =:= $' orelse
S =:= $[ orelse S =:= $( orelse S =:= ${ orelse S =:= $|)).
-define(LOC(Line, Column), [{line, Line}, {column, Column}]).
%% Spaces
-define(is_horizontal_space(S), (S =:= $\s orelse S =:= $\t)).
-define(is_vertical_space(S), (S =:= $\r orelse S =:= $\n)).
-define(is_space(S), (?is_horizontal_space(S) orelse ?is_vertical_space(S))).
%% Bidirectional control
%% Retrieved from https://trojansource.codes/trojan-source.pdf
-define(bidi(C), C =:= 16#202A orelse
C =:= 16#202B orelse
C =:= 16#202D orelse
C =:= 16#202E orelse
C =:= 16#2066 orelse
C =:= 16#2067 orelse
C =:= 16#2068 orelse
C =:= 16#202C orelse
C =:= 16#2069).
%% Unsupported newlines
%% https://www.unicode.org/reports/tr55/
%% CR (0x000D) is included: a bare CR not followed by LF is rejected.
%% Call sites must handle CRLF explicitly before the ?break check.
-define(break(C), C =:= 16#000B orelse
C =:= 16#000C orelse
C =:= 16#000D orelse
C =:= 16#0085 orelse
C =:= 16#2028 orelse
C =:= 16#2029).