Browse Source

SIMD fast path for skipping long multi-line comments. ~190MB/sec

master
A.Olokhtonov 2 years ago
parent
commit
ff1826e43e
  1. 18
      input.c
  2. 159
      main.c

18
input.c

@ -8,6 +8,8 @@ multi
line line
comment */ comment */
/* short com */
123 123
0707 0707
0xfAb 0xfAb
@ -73,8 +75,20 @@ L"\uffFF\xff"
[] ( ) { } . -> [] ( ) { } . ->
++ -- & * + - ~ ! ++ -- & * + - ~ !
/ % << >> < > <= >= / %
<<
>>
<
>
<=
>=
? : ; ... ? : ; ...
= *= /= %= += -= <<= = *= /= %= += -= <<=
, # ## , # ##
<: :> <% %> %: %:%: <:
:>
<%
%> %: %:%:
<stdio.h>
<stdio".h>

159
main.c

@ -10,8 +10,9 @@
#include <immintrin.h> #include <immintrin.h>
enum token_kind { #define MAX(a, b) ((a) > (b) ? (a) : (b))
enum token_kind {
TOKEN_KEYWORD, TOKEN_KEYWORD,
TOKEN_IDENTIFIER, TOKEN_IDENTIFIER,
@ -58,42 +59,30 @@ advance(struct str *s, int by)
static int static int
nondigit(struct str s) nondigit(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || (c == '_')); return(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || (c == '_'));
} }
return(0);
}
static int static int
digit(struct str s) digit(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return('0' <= c && c <= '9'); return('0' <= c && c <= '9');
} }
return(0);
}
static int static int
nonzero_digit(struct str s) nonzero_digit(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return('1' <= c && c <= '9'); return('1' <= c && c <= '9');
} }
return(0);
}
static int static int
octal_digit(struct str s) octal_digit(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return('0' <= c && c <= '7'); return('0' <= c && c <= '7');
} }
return(0);
}
static int static int
hexadecimal_prefix(struct str s) hexadecimal_prefix(struct str s)
@ -111,32 +100,23 @@ hexadecimal_prefix(struct str s)
static int static int
hexadecimal_digit(struct str s) hexadecimal_digit(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(digit(s) || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f')); return(digit(s) || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f'));
} }
return(0);
}
static int static int
unsigned_suffix(struct str s) unsigned_suffix(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c == 'u' || c == 'U'); return(c == 'u' || c == 'U');
} }
return(0);
}
static int static int
long_suffix(struct str s) long_suffix(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c == 'l' || c == 'L'); return(c == 'l' || c == 'L');
} }
return(0);
}
static int static int
long_long_suffix(struct str s) long_long_suffix(struct str s)
@ -154,22 +134,16 @@ long_long_suffix(struct str s)
static int static int
sign(struct str s) sign(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c == '+' || c == '-'); return(c == '+' || c == '-');
} }
return(0);
}
static int static int
floating_suffix(struct str s) floating_suffix(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c == 'f' || c == 'l' || c == 'F' || c == 'L'); return(c == 'f' || c == 'l' || c == 'F' || c == 'L');
} }
return(0);
}
static int static int
integer_suffix(struct str s) integer_suffix(struct str s)
@ -291,11 +265,7 @@ hex_quad(struct str s)
static int static int
universal_character_name(struct str s) universal_character_name(struct str s)
{ {
if (s.size < 2) { if (s.size >= 2 && s.text[0] == '\\') {
return(0);
}
if (s.text[0] == '\\') {
if (s.text[1] == 'u') { if (s.text[1] == 'u') {
advance(&s, 2); advance(&s, 2);
int hq = hex_quad(s); int hq = hex_quad(s);
@ -382,7 +352,6 @@ octal_constant(struct str s)
{ {
int start = s.size; int start = s.size;
if (s.size) {
if (s.text[0] == '0') { if (s.text[0] == '0') {
advance(&s, 1); advance(&s, 1);
@ -396,7 +365,6 @@ octal_constant(struct str s)
return(start - s.size); return(start - s.size);
} }
}
return(0); return(0);
} }
@ -471,7 +439,6 @@ fractional_constant(struct str s)
s.text += ds1; s.text += ds1;
s.size -= ds1; s.size -= ds1;
if (s.size) {
if (s.text[0] == '.') { if (s.text[0] == '.') {
advance(&s, 1); advance(&s, 1);
@ -482,7 +449,6 @@ fractional_constant(struct str s)
return(ds1 + ds2 + 1); return(ds1 + ds2 + 1);
} }
} }
}
return(0); return(0);
} }
@ -492,7 +458,6 @@ exponent_part(struct str s)
{ {
int start = s.size; int start = s.size;
if (s.size) {
if (s.text[0] == 'e' || s.text[0] == 'E') { if (s.text[0] == 'e' || s.text[0] == 'E') {
advance(&s, 1); advance(&s, 1);
@ -505,7 +470,6 @@ exponent_part(struct str s)
return(start - s.size); return(start - s.size);
} }
} }
}
return(0); return(0);
} }
@ -564,7 +528,7 @@ hexadecimal_fractional_constant(struct str s)
advance(&s, hds1); advance(&s, hds1);
if (s.size && s.text[0] == '.') { if (s.text[0] == '.') {
advance(&s, 1); advance(&s, 1);
int hds2 = hexadecimal_digit_sequence(s); int hds2 = hexadecimal_digit_sequence(s);
@ -582,7 +546,7 @@ binary_exponent_part(struct str s)
{ {
int start = s.size; int start = s.size;
if (s.size && (s.text[0] == 'p' || s.text[0] == 'P')) { if (s.text[0] == 'p' || s.text[0] == 'P') {
advance(&s, 1); advance(&s, 1);
int sgn = sign(s); int sgn = sign(s);
@ -673,36 +637,25 @@ c_char(struct str s)
return(sym); return(sym);
} }
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c != '\'' && c != '\\' && c != '\n');
}
return(0); return(c != '\'' && c != '\\' && c != '\n');
} }
static int static int
h_char(struct str s) h_char(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c != '\n' && c != '>'); return(c != '\n' && c != '>');
} }
return(0);
}
static int static int
q_char(struct str s) q_char(struct str s)
{ {
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c != '\n' && c != '\"'); return(c != '\n' && c != '\"');
} }
return(0);
}
static int static int
c_char_sequence(struct str s) c_char_sequence(struct str s)
{ {
@ -732,7 +685,7 @@ character_constant(struct str s)
int start = s.size; int start = s.size;
int ok = 0; int ok = 0;
if (s.size && s.text[0] == '\'') { if (s.text[0] == '\'') {
advance(&s, 1); advance(&s, 1);
ok = 1; ok = 1;
} else if (s.size >= 2 && s.text[0] == 'L' && s.text[1] == '\'') { } else if (s.size >= 2 && s.text[0] == 'L' && s.text[1] == '\'') {
@ -750,7 +703,7 @@ character_constant(struct str s)
int ccs = c_char_sequence(s); int ccs = c_char_sequence(s);
if (ccs) { if (ccs) {
advance(&s, ccs); advance(&s, ccs);
if (s.size && s.text[0] == '\'') { if (s.text[0] == '\'') {
advance(&s, 1); advance(&s, 1);
return(start - s.size); return(start - s.size);
} }
@ -777,8 +730,9 @@ static int
whitespace(struct str s) whitespace(struct str s)
{ {
int start = s.size; int start = s.size;
//int spaces = 0x20090a0d;
while (s.size && (s.text[0] == ' ' || s.text[0] == '\t' || s.text[0] == '\n' || s.text[0] == '\r')) { while (s.text[0] == ' ' || s.text[0] == '\t' || s.text[0] == '\n' || s.text[0] == '\r') {
advance(&s, 1); advance(&s, 1);
} }
@ -794,14 +748,10 @@ s_char(struct str s)
return(sym); return(sym);
} }
if (s.size) {
char c = s.text[0]; char c = s.text[0];
return(c != '\"' && c != '\\' && c != '\n'); return(c != '\"' && c != '\\' && c != '\n');
} }
return(0);
}
static int static int
s_char_sequence(struct str s) s_char_sequence(struct str s)
{ {
@ -832,14 +782,8 @@ encoding_prefix(struct str s)
return(2); return(2);
} }
if (s.size) {
char c = s.text[0]; char c = s.text[0];
if (c == 'u' || c == 'U' || c == 'L') { return(c == 'u' || c == 'U' || c == 'L');
return(1);
}
}
return(0);
} }
static int static int
@ -850,13 +794,13 @@ string_literal(struct str s)
int ep = encoding_prefix(s); int ep = encoding_prefix(s);
advance(&s, ep); advance(&s, ep);
if (s.size && s.text[0] == '\"') { if (s.text[0] == '\"') {
advance(&s, 1); advance(&s, 1);
int scs = s_char_sequence(s); int scs = s_char_sequence(s);
advance(&s, scs); advance(&s, scs);
if (s.size && s.text[0] == '\"') { if (s.text[0] == '\"') {
advance(&s, 1); advance(&s, 1);
return(start - s.size); return(start - s.size);
} }
@ -908,7 +852,6 @@ punctuator(struct str s)
} }
} }
if (s.size) {
char c = s.text[0]; char c = s.text[0];
if (c == '[' || c == ']' || c == '(' || c == ')' || if (c == '[' || c == ']' || c == '(' || c == ')' ||
c == '{' || c == '}' || c == '.' || c == '&' || c == '{' || c == '}' || c == '.' || c == '&' ||
@ -920,7 +863,6 @@ punctuator(struct str s)
{ {
return(1); return(1);
} }
}
return(0); return(0);
} }
@ -974,27 +916,27 @@ q_char_sequence(struct str s)
static int static int
header_name(struct str s) header_name(struct str s)
{ {
if (s.size && s.text[0] == '<') { if (s.text[0] == '<') {
advance(&s, 1); advance(&s, 1);
int hcs = h_char_sequence(s); int hcs = h_char_sequence(s);
if (hcs) { if (hcs) {
advance(&s, hcs); advance(&s, hcs);
if (s.size && s.text[0] == '>') { if (s.text[0] == '>') {
return(hcs + 2); return(hcs + 2);
} }
} }
} }
if (s.size && s.text[0] == '\"') { if (s.text[0] == '\"') {
advance(&s, 1); advance(&s, 1);
int qcs = q_char_sequence(s); int qcs = q_char_sequence(s);
if (qcs) { if (qcs) {
advance(&s, qcs); advance(&s, qcs);
if (s.size && s.text[0] == '\"') { if (s.text[0] == '\"') {
return(qcs + 2); return(qcs + 2);
} }
} }
@ -1028,6 +970,28 @@ comment(struct str s)
if (s.text[0] == '/' && s.text[1] == '*') { if (s.text[0] == '/' && s.text[1] == '*') {
/* multi-line comment */ /* multi-line comment */
advance(&s, 2);
__m128i mask = _mm_setr_epi8('*', '/', '*', '/', '*', '/', '*', '/', '*', '/', '*', '/', '*', '/', '*', '/');
__m128i mask_sus = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '*');
while (s.size > 16) {
__m128i chunk = _mm_loadu_si128((__m128i *)(s.text));
__m128i v1 = _mm_cmpeq_epi16(chunk, mask);
__m128i v2 = _mm_cmpeq_epi16(_mm_bslli_si128(chunk, 1), mask);
__m128i v3 = _mm_cmpeq_epi8(chunk, mask_sus);
__m128i v12 = _mm_or_si128(v1, v2);
__m128i v123 = _mm_or_si128(v12, v3);
if (!_mm_testz_si128(v123, v123)) {
break;
}
advance(&s, 16);
}
while (s.size) { while (s.size) {
if (s.size >= 2 && s.text[0] == '*' && s.text[1] == '/') { if (s.size >= 2 && s.text[0] == '*' && s.text[1] == '/') {
advance(&s, 2); advance(&s, 2);
@ -1057,36 +1021,42 @@ lex(char *text, int size)
if ((sym = comment(s))) { if ((sym = comment(s))) {
//printf("Comment: "); //printf("Comment: ");
} else if ((sym = constant(s))) { //printf("%.*s\n", sym, s.text);
//printf("Constant: "); } else {
int sym_constant = constant(s);
int sym_punctuator = punctuator(s);
int sym_string = string_literal(s);
int sym_header = header_name(s);
int sym_identifier = identifier(s);
sym = MAX(sym_constant, MAX(sym_punctuator, MAX(sym_string, MAX(sym_header, sym_identifier))));
#if 0 #if 0
} else if ((sym == pp_number(s))) { if (sym == sym_constant) {
printf("PP number: "); printf("Constant: ");
} else if (sym == sym_punctuator) {
printf("Punctuator: ");
} else if (sym == sym_string) {
printf("String: ");
} else if (sym == sym_header) {
printf("Header: ");
} else if (sym == sym_identifier) {
printf("Identifier: ");
}
#endif #endif
} else if ((sym = punctuator(s))) {
//printf("Punctuator: ");
} else if ((sym = string_literal(s))) {
//printf("String literal: ");
} else if ((sym = header_name(s))) {
//printf("Header name: ");
} else if ((sym = identifier(s))) {
//printf("Identifier: ");
} }
//printf("%.*s\n", sym, s.text);
if (sym) { if (sym) {
advance(&s, sym); advance(&s, sym);
} else if (s.size) { } else if (s.size == 1 && s.text[0] == '\0') {
break;
} else {
fprintf(stderr, "Error!\n"); fprintf(stderr, "Error!\n");
break; break;
} }
} }
return(NULL); return(NULL);
} }
@ -1169,6 +1139,13 @@ main(int argc, char **argv)
int size = (int) sb.st_size; int size = (int) sb.st_size;
char *data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); char *data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
if (size && data[size - 1] != '\n') {
fprintf(stderr, "No terminating new line. Fuck you!\n");
return(1);
}
data[size - 1] = '\0';
if (data == MAP_FAILED) { if (data == MAP_FAILED) {
perror("mmap"); perror("mmap");
return(1); return(1);

Loading…
Cancel
Save