tccpp : "tcc -E -P" : suppress empty lines

Also:
- regenerate all tests/pp/*.expect with gcc
- test "insert one space" feature
- test "0x1E-1" in asm mode case
- PARSE_FLAG_SPACES: ignore \f\v\r better
- tcc.h: move some things

tcc.h

@@ -324,12 +324,6 @@
 #define TOK_HASH_SIZE       16384 /* must be a power of two */
 #define TOK_ALLOC_INCR      512  /* must be a power of two */
 #define TOK_MAX_SIZE        4 /* token max size in int unit when stored in string */
-#define TOKSYM_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenSym in table_ident */
-#define TOKSTR_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenString instances */
-#define CSTR_TAL_SIZE       (256 * 1024) /* allocator for tiny CString instances */
-#define TOKSYM_TAL_LIMIT    256 /* prefer unique limits to distinguish allocators debug msgs */
-#define TOKSTR_TAL_LIMIT    128 /* 32 * sizeof(int) */
-#define CSTR_TAL_LIMIT      1024
 
 /* token symbol management */
 typedef struct TokenSym {
@@ -1021,42 +1015,15 @@ enum tcc_token {
 #undef DEF
 };
 
+/* keywords: tok >= TOK_IDENT && tok < TOK_UIDENT */
 #define TOK_UIDENT TOK_DEFINE
 
-/* space exlcuding newline */
+/* -------------------------------------------- */
-static inline int is_space(int ch)
-{
-    return ch == ' ' || ch == '\t' || ch == '\v' || ch == '\f' || ch == '\r';
-}
 
-static inline int isid(int c)
+#ifndef PUB_FUNC /* functions used by tcc.c but not in libtcc.h */
-{
-    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
-}
-
-static inline int isnum(int c)
-{
-    return c >= '0' && c <= '9';
-}
-
-static inline int isoct(int c)
-{
-    return c >= '0' && c <= '7';
-}
-
-static inline int toup(int c)
-{
-    return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c;
-}
-
-#ifndef PUB_FUNC
 # define PUB_FUNC
 #endif
 
-#ifdef TCC_PROFILE /* profile all functions */
-# define static
-#endif
-
 #ifdef ONE_SOURCE
 #define ST_INLN static inline
 #define ST_FUNC static
@@ -1067,6 +1034,10 @@ static inline int toup(int c)
 #define ST_DATA extern
 #endif
 
+#ifdef TCC_PROFILE /* profile all functions */
+# define static
+#endif
+
 /* ------------ libtcc.c ------------ */
 
 /* use GNU C extensions */
@@ -1240,6 +1211,23 @@ ST_FUNC int tcc_preprocess(TCCState *s1);
 ST_FUNC void skip(int c);
 ST_FUNC NORETURN void expect(const char *msg);
 
+/* space exlcuding newline */
+static inline int is_space(int ch) {
+    return ch == ' ' || ch == '\t' || ch == '\v' || ch == '\f' || ch == '\r';
+}
+static inline int isid(int c) {
+    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
+}
+static inline int isnum(int c) {
+    return c >= '0' && c <= '9';
+}
+static inline int isoct(int c) {
+    return c >= '0' && c <= '7';
+}
+static inline int toup(int c) {
+    return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c;
+}
+
 /* ------------ tccgen.c ------------ */
 
 ST_DATA Section *text_section, *data_section, *bss_section; /* predefined sections */
@@ -1422,49 +1410,23 @@ ST_FUNC void gen_vla_sp_save(int addr);
 ST_FUNC void gen_vla_sp_restore(int addr);
 ST_FUNC void gen_vla_alloc(CType *type, int align);
 
-static inline uint16_t read16le(unsigned char *p)
+static inline uint16_t read16le(unsigned char *p) {
-{
     return p[0] | (uint16_t)p[1] << 8;
 }
-
+static inline void write16le(unsigned char *p, uint16_t x) {
-static inline void write16le(unsigned char *p, uint16_t x)
+    p[0] = x & 255, p[1] = x >> 8 & 255;
-{
-    p[0] = x & 255;
-    p[1] = x >> 8 & 255;
 }
-
+static inline uint32_t read32le(unsigned char *p) {
-static inline uint32_t read32le(unsigned char *p)
+  return read16le(p) | (uint32_t)read16le(p + 2) << 16;
-{
-  return (p[0] | (uint32_t)p[1] << 8 |
-	  (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24);
 }
-
+static inline void write32le(unsigned char *p, uint32_t x) {
-static inline void write32le(unsigned char *p, uint32_t x)
+    write16le(p, x), write16le(p + 2, x >> 16);
-{
-    p[0] = x & 255;
-    p[1] = x >> 8 & 255;
-    p[2] = x >> 16 & 255;
-    p[3] = x >> 24 & 255;
 }
-
+static inline uint64_t read64le(unsigned char *p) {
-static inline uint64_t read64le(unsigned char *p)
+  return read32le(p) | (uint64_t)read32le(p + 4) << 32;
-{
-  return (p[0] | (uint64_t)p[1] << 8 |
-	  (uint64_t)p[2] << 16 | (uint64_t)p[3] << 24 |
-	  (uint64_t)p[4] << 32 | (uint64_t)p[5] << 40 |
-	  (uint64_t)p[6] << 48 | (uint64_t)p[7] << 56);
 }
-
+static inline void write64le(unsigned char *p, uint64_t x) {
-static inline void write64le(unsigned char *p, uint64_t x)
+    write32le(p, x), write32le(p + 4, x >> 32);
-{
-    p[0] = x & 255;
-    p[1] = x >> 8 & 255;
-    p[2] = x >> 16 & 255;
-    p[3] = x >> 24 & 255;
-    p[4] = x >> 32 & 255;
-    p[5] = x >> 40 & 255;
-    p[6] = x >> 48 & 255;
-    p[7] = x >> 56 & 255;
 }
 
 /* ------------ i386-gen.c ------------ */

tccgen.c

@@ -6334,7 +6334,7 @@ static int decl0(int l, int is_for_loop_init)
             }
             /* special test for old K&R protos without explicit int
                type. Only accepted when defining global data */
-            if (l == VT_LOCAL || tok < TOK_DEFINE)
+            if (l == VT_LOCAL || tok < TOK_UIDENT)
                 break;
             btype.t = VT_INT;
         }

tccpp.c

@@ -127,12 +127,19 @@ ST_FUNC void expect(const char *msg)
 #define TAL_DEBUG_PARAMS
 #else
 #define TAL_DEBUG 1
+//#define TAL_INFO 1 /* collect and dump allocators stats */
 #define tal_free(al, p) tal_free_impl(al, p, __FILE__, __LINE__)
 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size, __FILE__, __LINE__)
 #define TAL_DEBUG_PARAMS , const char *file, int line
 #define TAL_DEBUG_FILE_LEN 15
 #endif
-//#define TAL_INFO 1 /* collect and dump allocators stats */
+
+#define TOKSYM_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenSym in table_ident */
+#define TOKSTR_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenString instances */
+#define CSTR_TAL_SIZE       (256 * 1024) /* allocator for tiny CString instances */
+#define TOKSYM_TAL_LIMIT    256 /* prefer unique limits to distinguish allocators debug msgs */
+#define TOKSTR_TAL_LIMIT    128 /* 32 * sizeof(int) */
+#define CSTR_TAL_LIMIT      1024
 
 typedef struct TinyAlloc {
     size_t  limit;
@@ -3039,7 +3046,8 @@ static int next_argstream(Sym **nested_list, int can_read_stream, TokenString *w
                             break;
                         ch = ' ';
                     }
-                    tok_str_add(ws_str, ch);
+                    if (!(ch == '\f' || ch == '\v' || ch == '\r'))
+                        tok_str_add(ws_str, ch);
                     cinp();
                 }
             }
@@ -3569,15 +3577,13 @@ static void tok_print(const char *msg, const int *str)
 static void pp_line(TCCState *s1, BufferedFile *f, int level)
 {
     int d = f->line_num - f->line_ref;
+
     if (s1->dflag & 4)
 	return;
+
     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_NONE) {
-	if (level == 0 && f->line_ref && d) {
+        ;
-	    d = 1;
-	    goto simple;
-	}
     } else if (level == 0 && f->line_ref && d < 8) {
-simple:
 	while (d > 0)
 	    fputs("\n", s1->ppfp), --d;
     } else if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_STD) {

tests/pp/02.expect

@@ -1,6 +1,5 @@
 f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1);
-f(2 * (2+(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);
+f(2 * (2 +(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);
-
 char c[2][6] = { "hello", "" };
 f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1);
-f(2 * (2+(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);
+f(2 * (2 +(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);

tests/pp/13.expect

@@ -1,6 +1,3 @@
 # `modelist' label. Each video mode record looks like:
-
-
-
 .text
 endtext:

tests/pp/15.c

@@ -0,0 +1,18 @@
+// insert a space between two tokens if otherwise they
+// would form a single token when read back
+
+#define n(x) x
+
+return (n(long)n(double))d;
+return n(A)n(++)n(+)n(B);
+return n(A)n(+)n(++)n(B);
+return n(A)n(++)n(+)n(+)n(B);
+
+// not a hex float
+return n(0x1E)n(-1);
+
+// unlike gcc but correct
+XXX: return n(x)+n(x)-n(1)+n(1)-2;
+
+// unlile gcc, but cannot appear in valid C
+XXX: return n(x)n(x)n(1)n(2)n(x);

tests/pp/15.expect

@@ -0,0 +1,7 @@
+return (long double)d;
+return A+++B;
+return A+ ++B;
+return A+++ +B;
+return 0x1E -1;
+XXX: return x+x-1 +1 -2;
+XXX: return x x 1 2 x;

tests/pp/Makefile

@@ -40,8 +40,8 @@ DIFF_OPTS = -Nu -b -B -I "^\#"
 clean:
 	rm -vf *.output
 
-# 02.test : DIFF_OPTS += -w
+02.test : DIFF_OPTS += -w
-# 15.test : DIFF_OPTS += -w
+15.test : DIFF_OPTS += -I"^XXX:"
 
 # diff options:
 # -b ighore space changes

tests/tcctest.c

@@ -2573,6 +2573,10 @@ void asm_test(void)
     unsigned int val;
 
     printf("inline asm:\n");
+
+    // parse 0x1E-1 as 3 tokens in asm mode
+    asm volatile ("mov $0x1E-1,%eax");
+
     /* test the no operand case */
     asm volatile ("xorl %eax, %eax");