@@ -5,3 +5,5 @@ tests_utf8

 *.log

 re.valgrind

 sdl.supp

+NormalizationTest.txt

+reference*.png

@@ -23,11 +23,13 @@

 #include "re_data.h"

 #include "sha3/sha3.h"

+/* DEFAULT CHUNKSIZE: 1024 */

+//#define CHUNKSIZE 65536

 //#define CHUNKSIZE 32768

 //#define CHUNKSIZE 4096

-#define CHUNKSIZE 1024

+//#define CHUNKSIZE 1024

 //#define CHUNKSIZE 160

-//#define CHUNKSIZE 16

+#define CHUNKSIZE 16

 #define UNDOBLOCK 1024

 #define ADDNBLOCK 1024

 #define UNDOGROWSIZE (256*1024)

@@ -100,7 +100,6 @@ typedef struct redata_t {

 redata_t *redata_init( int (*pluginregisterfn)(redata_t *redata, redata_plugin_t *slot), ...);

 void redata_free(redata_t *redata);

-

 void redata_idleproc(redata_t *redata, char *filename);

 int redata_config_chunkdatasize(redata_t *redata, int chunkdatasize);

@@ -22,19 +22,26 @@

 #include "re_plugin_highlighter.h"

 #define PLUGINNAME "highlighter"

+#define HLINETBLOCK (16*1024)

+#define BUFBLOCK (64*1024)

 #define HIGHLIGHTERGROWSIZE (256*1024)

 #define DEFAULTCOLOR "\x80\x80\x80\xff"

 typedef enum colorsenum_t {

         color_normal=0,

-        color_define,

-        color_definestring,

+        color_directive,

+        color_directivekeyword,

+        color_directivestring,

+        color_directiveinclude,

+        color_directiveincludestring,

         color_keyword,

         color_string,

         color_multilinecomment,

         color_linecomment,

         color_number,

+        color_operator,

+        color_symbol,

         color_endenum,

 } colorsenum_t;

@@ -42,13 +49,16 @@ typedef enum colorsenum_t {

 static int redata_highlighter_add(redata_t *redata, redata_plugin_t *slot, undo_t *undo);

 static int redata_highlighter_unadd(redata_t *redata, redata_plugin_t *slot, undo_t *undo);

 static int redata_highlighter_add_or_unadd(redata_t *redata, redata_plugin_t *slot, undo_t *undo, int is_unadd);

-static int redata_highlighter_commit(redata_t *redata, redata_plugin_t *slot,char *filename);

 static int redata_highlighter_postload(redata_t *redata, redata_plugin_t *slot,char *filename);

 static int hl_invalidatefrom(redata_t *redata, highlighter_t *hl, int nline);

 hcolor_t *hl_initcolors(int *ncolors, /* int color, char *colordef, */ ...);

 static highlighter_t *hl_getplugin(redata_t *redata);

 static int hl_doline(redata_t *redata, highlighter_t *hl, int nline);

+int hl_C_getkeywords(char ***keywords,int *nkeywords, int *maxlen);

+int hl_C_getdirectives(char ***directives,int *ndirectives, int *maxlen);

+int hl_searchlist(char **wordlist, int wordlistlen, char *word, int wordlen, int *res);

+linecolor_t *hl_addtolinecolor(int *opaque, highlighter_t *hl,linecolor_t *linecolor,int posoff, int color);

 int

 redata_highlighter_register(redata_t *redata, redata_plugin_t *slot)

@@ -60,13 +70,18 @@ redata_highlighter_register(redata_t *redata, redata_plugin_t *slot)

                 return(-1);

         colors=hl_initcolors(&ncolors,

                              color_normal,"\x38\x17\x1e\xff",

-                             color_define,"\x0d\x2b\x04\xff",

-                             color_definestring,"\x07\x20\3b\xff",

+                             color_directive,"\x38\x4b\x00\xff",

+                             color_directivekeyword,"\x63\x7d\x16\xff",

+                             color_directivestring,"\x07\x20\3b\xff",

+                             color_directiveinclude,"\x16\x63\x7d\xff",

+                             color_directiveincludestring,"\x6\x13\x2d\xff",

                              color_keyword,"\x9d\x15\x00\xff",

                              color_string,"\x68\x00\x01\xff",

                              color_multilinecomment,"\x4f\x40\x57\xff",

-                             color_linecomment,"\xaa\x8c\xcd\xff",

+                             color_linecomment,"\xc6\x8c\xa4\xff",

                              color_number,"\x3b\x10\x35\xff",

+                             color_operator,"\xaa\x8c\xcd\xff",

+                             color_symbol,"\x69\x2a\x44\xff",

                              color_endenum,DEFAULTCOLOR,

                              -1);

         if(colors==NULL || (hl=malloc(sizeof(highlighter_t)))==NULL) {

@@ -84,7 +99,6 @@ redata_highlighter_register(redata_t *redata, redata_plugin_t *slot)

         slot->postload=redata_highlighter_postload;

         slot->add=redata_highlighter_add;

         slot->unadd=redata_highlighter_unadd;

-        slot->commit=redata_highlighter_commit;

         slot->userptr=hl;

         return(0);

 }

@@ -104,6 +118,12 @@ redata_highlighter_unregister(redata_t *redata, redata_plugin_t *slot, char *fil

         if(hl->lines!=NULL)

                 free(hl->lines),hl->lines=NULL;

         hl->sizelines=hl->usedlines=0;

+        if(hl->keywordbuf!=NULL)

+                free(hl->keywordbuf),hl->keywordbuf=NULL;

+        hl->sizekeywordbuf=hl->usedkeywordbuf=0;

+        if(hl->directivebuf!=NULL)

+                free(hl->directivebuf),hl->directivebuf=NULL;

+        hl->sizedirectivebuf=hl->useddirectivebuf=0;

         if(slot->userptr!=NULL)

                 free(slot->userptr),slot->userptr=NULL;

         return(0);

@@ -131,7 +151,7 @@ redata_highlighter_getline(redata_t *redata, int line, int *nlinecolors)

         if(line<0 || line>=hl->usedlines)

                 return(NULL);

         if(nlinecolors!=NULL)

-                *nlinecolors=(hl->lines[line].len)/sizeof(hcolor_t);

+                *nlinecolors=(hl->lines[line].len)/sizeof(linecolor_t);

         return((linecolor_t *) (hl->buf+hl->lines[line].off));

 }

@@ -158,6 +178,12 @@ redata_highlighter_add_or_unadd(redata_t *redata, redata_plugin_t *slot, undo_t

         stack=redata_getstack(redata,undo);

         if(redata==NULL || slot==NULL || hl==NULL || undo==NULL || stack==NULL || slot->active==0)

                 return(-1); /* sanity check failed */

+        if(hl->usedlines==0) {

+#if 1

+fprintf(stderr,"INVALIDATING USEDLINELINES==0 (nothing to do)\n");

+#endif

+                return(0); /* nothing to do */

+        }

         /* get the first pos of the operation */

         pos=undo->posorig;

         if(undo->type=='D')

@@ -171,21 +197,15 @@ redata_highlighter_add_or_unadd(redata_t *redata, redata_plugin_t *slot, undo_t

                         break;

                 }

         }

+        /* special case: check if pos is inside last line */

+        if(nline>=hl->usedlines && (hl->lines[hl->usedlines-1].pos+hl->lines[hl->usedlines-1].len)>=pos)

+                nline=hl->usedlines-1;

         /* invalidate from this line on */

+        nline=(nline<0)?0:nline;

         hl_invalidatefrom(redata,hl,nline);

-        return(0);

-}

-

-static int

-redata_highlighter_commit(redata_t *redata, redata_plugin_t *slot,char *filename)

-{

-        highlighter_t *hl=(highlighter_t *) ((slot!=NULL)?(slot->userptr):NULL);

-        if(redata==NULL || hl==NULL || filename==NULL)

-                return(-1); /* sanity check failed */

-        if(hl->flag_doneall)

-                return(0);

-        /* calc highlight of one additional line */

-        hl_doline(redata, hl, hl->usedlines);

+#if 1

+fprintf(stderr,"INVALIDATING LINE %i\n",nline);

+#endif

         return(0);

 }

@@ -274,12 +294,582 @@ hl_getplugin(redata_t *redata)

         return(hl);

 }

-

 static int

 hl_doline(redata_t *redata, highlighter_t *hl, int nline)

 {

+        int i;

+        long realpos;

+        long pos;

+        char *ptr;

+        int len;

+        hline_t *line;

+        long redataused;

+        int has_nl;

+        int has_next;

+        char **keywords;

+        int nkeywords;

+        char **directives;

+        int ndirectives;

+        int maxlenkeywords,maxlendirectives;

+        linecolor_t *linecolor;

+        int prev_char;

+        int prev_char_mask;

+        int cant_define;

+        int cant_directivekeyword;

+        int is_directiveinclude;

+        int opaque;

+        int posoffset;

+        enum {

+                mode_whatever=0,

+                mode_in_multilinecomment,

+                mode_in_linecomment,

+                mode_in_directive,

+                mode_in_string,

+                mode_in_directivestring,

+        } mode,mlcprevmode;

+        if(redata==NULL || hl==NULL || nline<0)

+                return(-1); /* sanoty check failed */

+        if(hl->usedlines>nline)

+                return(0); /* nothing to do */

+        if(hl_C_getkeywords(&keywords,&nkeywords,&maxlenkeywords)==-1)

+                return(-1); /* couldn't get keyword list */

+        if(hl_C_getdirectives(&directives,&ndirectives,&maxlendirectives)==-1)

+                return(-1); /* couldn't get directive list */

+        /* make sure keywordbuf is large enough */

+        if(maxlenkeywords>hl->sizekeywordbuf) {

+                char *newkeywordbuf;

+                int newsize;

+                newsize=maxlenkeywords;

+                if((newkeywordbuf=realloc(hl->keywordbuf,newsize))==NULL)

+                        return(-1); /* insufficient memory */

+                memset(newkeywordbuf,0,newsize);

+                hl->keywordbuf=newkeywordbuf;

+                hl->sizekeywordbuf=newsize;

+                hl->usedkeywordbuf=0;

+        }

+        /* make sure directivebuf is large enough */

+        if(maxlendirectives>hl->sizedirectivebuf) {

+                char *newdirectivebuf;

+                int newsize;

+                newsize=maxlendirectives;

+                if((newdirectivebuf=realloc(hl->directivebuf,newsize))==NULL)

+                        return(-1); /* insufficient memory */

+                memset(newdirectivebuf,0,newsize);

+                hl->directivebuf=newdirectivebuf;

+                hl->sizedirectivebuf=newsize;

+                hl->useddirectivebuf=0;

+        }

+        /* make sure we have enough hline_t structs */

+        if(hl->sizelines<=nline) {

+                hline_t *newlines;

+                int newsize=(nline+1+HLINETBLOCK)/HLINETBLOCK;

+                newsize*=HLINETBLOCK;

+                if((newlines=realloc(hl->lines,newsize*sizeof(hline_t)))==NULL)

+                        return(-1); /* insufficient memory */

+                hl->lines=newlines;

+                memset(hl->lines+hl->sizelines,0,(newsize-hl->sizelines)*sizeof(hline_t));

+                hl->sizelines=newsize;

+        }

+        /* make sure we have previous lines highlighted */

+        for(i=hl->usedlines;i<nline;i++) {

+#if 0

+fprintf(stderr,"Recursing from %i to %i\n",nline,i);

+#endif

+                if(hl_doline(redata,hl,i)==-1)

+                        return(-1); /* error highlighting line */

+        }

+        hl_invalidatefrom(redata,hl,nline);

+        line=hl->lines+nline;

+        if(redata_linecol2pos(redata,nline,0,&realpos,NULL)==-1)

+                return(-1); /* couldn't get line pos */

         /* NOTE: here comes the highlighter */

-#warning XXX TODO: implement this

-        return(-1);

+        line->pos=realpos;

+        line->off=(nline==0)?0:hl->lines[nline-1].off+hl->lines[nline-1].len;

+        line->len=0;

+        line->endingmode=mode_whatever;

+        redataused=redata_getused(redata);

+        mode=(nline>0)?line[-1].endingmode:mode_whatever;

+        cant_define=0;

+        cant_directivekeyword=0;

+        is_directiveinclude=0;

+        opaque=0;

+        linecolor=hl_addtolinecolor(&opaque,hl,NULL,color_normal,0);

+        posoffset=0;

+#if 0

+fprintf(stderr,"Doing line %i pos:%li\n",nline,line->pos);

+#endif

+        prev_char='\0';

+        prev_char_mask=0xff;

+        hl->usedkeywordbuf=0;

+        hl->useddirectivebuf=0;

+        mlcprevmode=mode_whatever;

+        do {

+                if(redata_line_rawinfo(redata,realpos+posoffset,&pos,&ptr,&len,NULL)==-1)

+                        return(-1); /* couldn't get line data */

+                has_nl=((len>0 && ptr[len-1]=='\n')?1:0);

+                has_next=(len==0)?1:(ptr[len-1]=='\n')?0:1;

+                /* special case: line with only a newline */

+                if(posoffset==0 && has_nl==1 && len==1) {

+                        /* delete the existing linecolor and break */

+                        line->len=0;

+                        break;

+                }

+                /* iterate */

+#if 0

+fprintf(stderr,"Line %i chunk at posoffset %i\n",nline, posoffset);

+#endif

+#warning FALTA PONER DE UN COLOR DIFERENTE lo de <nombre.h> de los #include (pero no de los #define!)

+                for(i=0;i<(len-has_nl);prev_char=(ptr[i]&prev_char_mask),prev_char_mask=0xff,i++) {

+#if 0

+if(nline==1)

+        fprintf(stderr,"ptr[%i]='%c' prev_char='%s%c'\n",i,ptr[i],(prev_char=='\0')?"\\":"",(prev_char=='\0')?'0':prev_char);

+#endif

+                        /* special case: keyword ends in a change of mode */

+                        if(mode!=mode_whatever && hl->usedkeywordbuf>0) {

+                                if(hl_searchlist(keywords,nkeywords,hl->keywordbuf,hl->usedkeywordbuf,NULL)==0) {

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->keywordbufstart,color_keyword);

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->keywordbufstart+hl->usedkeywordbuf-1,color_keyword);

+                                }

+                                hl->usedkeywordbuf=0;

+                        }

+                        /* end of special case */

+                        if(mode==mode_whatever) {

+                                if(prev_char=='\\' || ptr[i]=='\\') {

+                                        /* escape char mark or escaped char */

+                                        continue;

+                                }

+                                if(prev_char=='/' && ptr[i]=='/') {

+                                        mode=mode_in_linecomment;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i-1,color_linecomment);

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_linecomment);

+                                        continue;

+                                }

+                                if(prev_char=='/' && ptr[i]=='*') {

+                                        mlcprevmode=mode;

+                                        mode=mode_in_multilinecomment;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i-1,color_multilinecomment);

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_multilinecomment);

+                                        continue;

+                                }

+                                if(ptr[i]=='\"') {

+                                        mode=mode_in_string;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_string);

+                                        continue;

+                                }

+                                if(!cant_define && ptr[i]=='#') {

+                                        mode=mode_in_directive;

+                                        is_directiveinclude=0;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_directive);

+                                        continue;

+                                }

+                                /* keyword detection */

+                                if(hl->usedkeywordbuf==0 && ((ptr[i]>='a' && ptr[i]<='z') || (ptr[i]>='A' && ptr[i]<='Z') || ptr[i]=='_')) {

+                                        cant_define=1;

+                                        hl->keywordbuf[hl->usedkeywordbuf++]=ptr[i];

+                                        hl->keywordbufstart=posoffset+i;

+                                } else if(hl->usedkeywordbuf>0 && ((ptr[i]>='0' && ptr[i]<='9') || (ptr[i]>='a' && ptr[i]<='z') || (ptr[i]>='A' && ptr[i]<='Z') || ptr[i]=='_')) {

+                                        if(hl->usedkeywordbuf<hl->sizekeywordbuf) {

+                                                hl->keywordbuf[hl->usedkeywordbuf++]=ptr[i];

+                                        } else {

+                                                hl->keywordbuf[0]='\0'; /* too long */

+                                        }

+                                }

+                                if(hl->usedkeywordbuf>0 && !((ptr[i]>='0' && ptr[i]<='9') || (ptr[i]>='a' && ptr[i]<='z') || (ptr[i]>='A' && ptr[i]<='Z') || ptr[i]=='_')) {

+                                        if(hl_searchlist(keywords,nkeywords,hl->keywordbuf,hl->usedkeywordbuf,NULL)==0) {

+                                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->keywordbufstart,color_keyword);

+                                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->keywordbufstart+hl->usedkeywordbuf-1,color_keyword);

+                                        }

+                                        hl->usedkeywordbuf=0;

+

+                                }

+                                /* keyword detection */

+                                if(strchr("0123456789",ptr[i])!=NULL) {

+                                        cant_define=1;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_number);

+                                        continue;

+                                }

+                                if(strchr("<>=+-%*/!|&^",ptr[i])!=NULL) {

+                                        cant_define=1;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_operator);

+                                        continue;

+                                }

+                                if(strchr("(){}[],;",ptr[i])!=NULL) {

+                                        cant_define=1;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_symbol);

+                                        continue;

+                                }

+                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_normal);

+                                continue;

+                        }

+                        if(mode==mode_in_multilinecomment) {

+                                if(prev_char=='\\' || ptr[i]=='\\') {

+                                        continue;

+                                }

+                                if(prev_char=='*' && ptr[i]=='/') {

+                                        prev_char_mask=0x00;

+                                        mode=mlcprevmode;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_multilinecomment);

+                                        continue;

+                                }

+                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_multilinecomment);

+                                continue;

+                        }

+                        if(mode==mode_in_directive) {

+                                if(prev_char=='\\' || ptr[i]=='\\') {

+                                        continue;

+                                }

+                                if(prev_char=='/' && ptr[i]=='/') {

+                                        mode=mode_in_linecomment;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i-1,color_linecomment);

+                                         linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_linecomment);

+                                        continue;

+                                }

+                                if(prev_char=='/' && ptr[i]=='*') {

+                                        mlcprevmode=mode;

+                                        mode=mode_in_multilinecomment;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i-1,color_multilinecomment);

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_multilinecomment);

+                                        continue;

+                                }

+                                /* directive detection */

+                                if(!cant_directivekeyword && hl->useddirectivebuf==0 && ((ptr[i]>='a' && ptr[i]<='z') || (ptr[i]>='A' && ptr[i]<='Z') || ptr[i]=='_')) {

+                                        hl->directivebuf[hl->useddirectivebuf++]=ptr[i];

+                                        hl->directivebufstart=posoffset+i;

+                                } else if(hl->useddirectivebuf>0 && ((ptr[i]>='0' && ptr[i]<='9') || (ptr[i]>='a' && ptr[i]<='z') || (ptr[i]>='A' && ptr[i]<='Z') || ptr[i]=='_')) {

+                                        if(hl->useddirectivebuf<hl->sizedirectivebuf) {

+                                                hl->directivebuf[hl->useddirectivebuf++]=ptr[i];

+                                        } else {

+                                                hl->directivebuf[0]='\0'; /* too long */

+                                                cant_directivekeyword=1;

+                                        }

+                                }

+                                if(hl->useddirectivebuf>0 && !((ptr[i]>='0' && ptr[i]<='9') || (ptr[i]>='a' && ptr[i]<='z') || (ptr[i]>='A' && ptr[i]<='Z') || ptr[i]=='_')) {

+                                        int nfound;

+                                        if(hl_searchlist(directives,ndirectives,hl->directivebuf,hl->useddirectivebuf,&nfound)==0) {

+                                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->directivebufstart,color_directivekeyword);

+                                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->directivebufstart+hl->useddirectivebuf-1,color_directivekeyword);

+                                                if(strcmp(directives[nfound],"include")==0)

+                                                        is_directiveinclude=1;

+                                        }

+                                        hl->useddirectivebuf=0;

+                                        cant_directivekeyword=1;

+                                }

+                                if(ptr[i]!=' ' && ptr[i]!='\t')

+                                        cant_directivekeyword=1;

+                                /* directive detection */

+                                if(ptr[i]=='\"') {

+                                        mode=mode_in_directivestring;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,(is_directiveinclude==0)?color_directivestring:color_directiveincludestring);

+                                        continue;

+                                }

+                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,(is_directiveinclude==0)?color_directive:color_directiveinclude);

+                                continue;

+                        }

+                        if(mode==mode_in_directivestring) {

+                                if(prev_char=='\\' || ptr[i]=='\\') {

+                                        continue;

+                                }

+                                if(ptr[i]=='\"') {

+                                        mode=mode_in_directive;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,(is_directiveinclude==0)?color_directivestring:color_directiveincludestring);

+                                        continue;

+                                }

+                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,(is_directiveinclude==0)?color_directivestring:color_directiveincludestring);

+                                continue;

+                        }

+                        if(mode==mode_in_string) {

+                                if(prev_char=='\\' || ptr[i]=='\\') {

+                                        continue;

+                                }

+                                if(ptr[i]=='\"') {

+                                        mode=mode_whatever;

+                                        linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_string);

+                                        continue;

+                                }

+                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_string);

+                                continue;

+                        }

+                        if(mode==mode_in_linecomment) {

+                                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,posoffset+i,color_linecomment);

+                                continue;

+                        }

+                }

+                posoffset+=len;

+        } while(has_next!=0 && (pos+posoffset)<redataused);

+        /* special case: keyword ends at end-of-line */

+        if(mode==mode_whatever && hl->usedkeywordbuf>0 && hl_searchlist(keywords,nkeywords,hl->keywordbuf,hl->usedkeywordbuf,NULL)==0) {

+                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->keywordbufstart,color_keyword);

+                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->keywordbufstart+hl->usedkeywordbuf-1,color_keyword);

+        }

+        /* end of special case */

+        /* special case: deirectivekeyword ends at end-of-line */

+        if(mode==mode_in_directive && hl->useddirectivebuf>0 && hl_searchlist(directives,ndirectives,hl->directivebuf,hl->useddirectivebuf,NULL)==0) {

+                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->directivebufstart,color_directivekeyword);

+                linecolor=hl_addtolinecolor(&opaque,hl,linecolor,hl->directivebufstart+hl->useddirectivebuf-1,color_directivekeyword);

+        }

+        /* end of special case */

+        if(linecolor==NULL)

+                return(-1);

+        if((prev_char=='\\' && mode!=mode_in_linecomment) || mode==mode_in_multilinecomment)

+                line->endingmode=mode;

+        hl->usedlines++;

+        return(0);

+}

+

+

+int

+hl_C_getkeywords(char ***keywords,int *nkeywords, int *maxlen)

+{

+        static int init=0;

+        static int staticnkeywords=0;

+        static int staticmaxlen=0;

+        static const char *C_keywords[]={

+"auto",

+"break",

+"case",

+"char",

+"const",

+"continue",

+"default",

+"do",

+"double",

+"else",

+"enum",

+"extern",

+"float",

+"for",

+"goto",

+"if",

+"inline",

+"int",

+"long",

+"register",

+"restrict",

+"return",

+"short",

+"signed",

+"sizeof",

+"static",

+"struct",

+"switch",

+"typedef",

+"union",

+"unsigned",

+"void",

+"volatile",

+"while",

+"_Alignas",

+"_Alignof",

+"_Atomic",

+"_Bool",

+"_Complex",

+"_Generic",

+"_Imaginary",

+"_Noreturn",

+"_Static_assert",

+"_Thread_local",

+NULL

+};

+        if(keywords==NULL || nkeywords==NULL)

+                return(-1);

+        if(init==0) {

+                int k,l,maxl;

+                for(k=0,maxl=0;C_keywords[k]!=NULL;k++) {

+                        l=strlen(C_keywords[k]);

+                        maxl=(l>maxl)?l:maxl;

+                }

+                staticnkeywords=k;

+                staticmaxlen=maxl;

+                init=1;

+        }

+        *keywords=(char **)C_keywords;

+        *nkeywords=staticnkeywords;

+        if(maxlen!=NULL)

+                *maxlen=staticmaxlen;

+        return(0);

+}

+

+int

+hl_C_getdirectives(char ***directives,int *ndirectives, int *maxlen)

+{

+        static int init=0;

+        static int staticndirectives=0;

+        static int staticmaxlen=0;

+        static const char *C_directives[]={

+"define",

+"undef",

+"include",

+"if",

+"ifdef",

+"ifndef",

+"else",

+"elif",

+"endif",

+"line",

+"error",

+"pragma",

+};

+        if(directives==NULL || ndirectives==NULL)

+                return(-1);

+        if(init==0) {

+                int k,l,maxl;

+                for(k=0,maxl=0;C_directives[k]!=NULL;k++) {

+                        l=strlen(C_directives[k]);

+                        maxl=(l>maxl)?l:maxl;

+                }

+                staticndirectives=k;

+                staticmaxlen=maxl;

+                init=1;

+        }

+        *directives=(char **)C_directives;

+        *ndirectives=staticndirectives;

+        if(maxlen!=NULL)

+                *maxlen=staticmaxlen;

+        return(0);

+}

+

+int

+hl_searchlist(char **wordlist, int wordlistlen, char *word, int wordlen, int *res)

+{

+        int k;

+        if(wordlist==NULL || wordlistlen<0 || word==NULL || wordlen<0)

+                return(-1); /* sanity check failed */

+        for(k=0;k<wordlistlen;k++) {

+                if(memcmp(wordlist[k],word,wordlen)==0 && (wordlist[k])[wordlen]=='\0') {

+                        if(res!=NULL)

+                                *res=k;

+                        return(0); /* word found */

+                }

+        }

+        return(-1); /* word not found */

+}

+

+

+linecolor_t *

+hl_addtolinecolor(int *opaque, highlighter_t *hl,linecolor_t *linecolor,int posoff,int color)

+{

+        hline_t *line;

+        if(opaque==NULL || *opaque<0 || hl==NULL || (posoff!=0 && linecolor==NULL) || color<0 || posoff<0)

+                return(NULL); /* sanity check failed */

+#if 0

+#define DEBUGLINE 12

+if(hl->usedlines==DEBUGLINE) {

+int i,k;

+int l=hl->lines[hl->usedlines].len;

+linecolor_t *lc=(linecolor_t *) (hl->buf+hl->lines[hl->usedlines].off);

+fprintf(stderr,"addtolinecolor pre.: nline:%i *opaque:%i posoff:%i color:%i current:\"",hl->usedlines,*opaque,posoff,color);

+for(k=0;k<l;k+=sizeof(linecolor_t),lc++) {

+        for(i=0;i<lc->len;i++)

+                fprintf(stderr,"%x",lc->color);

+}

+hl->keywordbuf[hl->usedkeywordbuf]='\0';

+hl->directivebuf[hl->useddirectivebuf]='\0';

+fprintf(stderr,"\" (%i%s%s%s,%i%s%s%s)\n",

+hl->usedkeywordbuf,

+(hl->usedkeywordbuf==0)?"":" \"",

+(hl->usedkeywordbuf==0)?"":hl->keywordbuf,

+(hl->usedkeywordbuf==0)?"":"\"",

+hl->useddirectivebuf,

+(hl->useddirectivebuf==0)?"":" \"",

+(hl->useddirectivebuf==0)?"":hl->directivebuf,

+(hl->useddirectivebuf==0)?"":"\""

+);

+}

+#endif

+        /* make sure there is a space for a linecolor_t in buf */

+        if((hl->usedbuf+sizeof(linecolor_t))>=hl->sizebuf) {

+                char *newbuf;

+                int newsize=(hl->sizebuf+sizeof(linecolor_t)+BUFBLOCK-1)/BUFBLOCK;

+                newsize*=BUFBLOCK;

+                if((newbuf=realloc(hl->buf,newsize))==NULL)

+                        return(NULL); /* insufficient memory */

+                hl->buf=newbuf;

+                memset(hl->buf+hl->sizebuf,0,(newsize-hl->sizebuf));

+                hl->sizebuf=newsize;

+        }

+        line=hl->lines+hl->usedlines;

+        /* posoff==0 means "do init" */

+        if(posoff==0) {

+                line->off=(hl->usedlines==0)?0:hl->lines[hl->usedlines-1].off+hl->lines[hl->usedlines-1].len;

+                line->len=sizeof(linecolor_t);

+                linecolor=(linecolor_t *) (hl->buf+line->off);

+                linecolor->len=1;

+                linecolor->color=color;

+                *opaque=1;

+#if 0

+if(hl->usedlines==DEBUGLINE) {

+int i,k;

+int l=hl->lines[hl->usedlines].len;

+linecolor_t *lc=(linecolor_t *) (hl->buf+hl->lines[hl->usedlines].off);

+fprintf(stderr,"addtolinecolor post: nline:%i *opaque:%i posoff:%i color:%i         \"",hl->usedlines,*opaque,posoff,color);

+for(k=0;k<l;k+=sizeof(linecolor_t),lc++) {

+        for(i=0;i<lc->len;i++)

+                fprintf(stderr,"%x",lc->color);

+}

+fprintf(stderr,"\"\n");

+}

+#endif

+                return(linecolor);

+        }

+        /* if posoff was already done, truncate */

+        if(posoff<*opaque) {

+                int delta,maxdelta;

+                int l;

+                maxdelta=line->len;

+                line->len=0;

+                for(delta=0,l=0;delta<maxdelta;l+=linecolor->len,delta+=sizeof(linecolor_t)) {

+                        linecolor=(linecolor_t *) (hl->buf+line->off+delta);

+                        line->len=delta;

+                        if((l+linecolor->len)>=posoff) {

+                                line->len+=sizeof(linecolor_t);

+                                linecolor->len=posoff-l;

+                                *opaque=posoff;

+                                break;

+                        }

+                }

+                hl->usedbuf=line->off+line->len;

+        }

+        linecolor=(linecolor_t *) (hl->buf+line->off+line->len);

+        /* if we have not changed color, expand */

+        if(linecolor[-1].color==color) {

+                int added=(posoff-*opaque)+1;

+                linecolor[-1].len+=added;

+                *opaque+=added;

+#if 0

+if(hl->usedlines==DEBUGLINE) {

+int i,k;

+int l=hl->lines[hl->usedlines].len;

+linecolor_t *lc=(linecolor_t *) (hl->buf+hl->lines[hl->usedlines].off);

+fprintf(stderr,"addtolinecolor post: nline:%i *opaque:%i posoff:%i color:%i         \"",hl->usedlines,*opaque,posoff,color);

+for(k=0;k<l;k+=sizeof(linecolor_t),lc++) {

+        for(i=0;i<lc->len;i++)

+                fprintf(stderr,"%x",lc->color);

+}

+fprintf(stderr,"\"\n");

 }

+#endif

+                return(linecolor-1);

+        }

+        /* add new linecolor */

+        linecolor->len=1;

+        linecolor->color=color;

+        *opaque+=1;

+        line->len+=sizeof(linecolor_t);

+        hl->usedbuf+=sizeof(linecolor_t);

+#if 0

+if(hl->usedlines==DEBUGLINE) {

+int i,k;

+int l=hl->lines[hl->usedlines].len;

+linecolor_t *lc=(linecolor_t *) (hl->buf+hl->lines[hl->usedlines].off);

+fprintf(stderr,"addtolinecolor post: nline:%i *opaque:%i posoff:%i color:%i         \"",hl->usedlines,*opaque,posoff,color);

+for(k=0;k<l;k+=sizeof(linecolor_t),lc++) {

+        for(i=0;i<lc->len;i++)

+                fprintf(stderr,"%x",lc->color);

+}

+fprintf(stderr,"\"\n");

+}

+#endif

+        return(linecolor);

+}

+

@@ -22,6 +22,7 @@ typedef struct line_t {

         long pos;

         int off; /* offset int buf */

         int len; /* size used in buf */

+        int endingmode;

 } hline_t;

 typedef struct hcolor_t {

@@ -38,6 +39,14 @@ typedef struct highlighter_t {

         int sizecolors;

         hcolor_t *colors;

         int flag_doneall;

+        int sizekeywordbuf;

+        int usedkeywordbuf;

+        char *keywordbuf;

+        int keywordbufstart;

+        int sizedirectivebuf;

+        int useddirectivebuf;

+        char *directivebuf;

+        int directivebufstart;

 } highlighter_t;

 int redata_highlighter_register(redata_t *redata, redata_plugin_t *slot);

@@ -16,6 +16,35 @@

 #include <time.h>

 #include "re_data.h"

+#if 0

+#define INITIALCHUNK 20

+#define MAXDUMP 25

+static void

+redata_debug_chunkdump(redata_t *redata, char *title)

+{

+        int m,k;

+        char c;

+        title=(title==NULL)?"":title;

+        fprintf(stderr,"%s:CHUNKDUMP (sizechunks:%i)\n",title,redata->sizechunks);

+        for(m=INITIALCHUNK;m<redata->sizechunks && m<MAXDUMP;m++) {

+                fprintf(stderr,"%s:chunk[%i] len:%-5i data:\"",title,m,redata->chunks[m]->useddata);

+                for(k=0;k<redata->chunks[m]->useddata;k++) {

+                        c=redata->chunks[m]->data[k];

+                        if(c=='\n' || c=='\0')

+                                fprintf(stderr,"\\%c",(c=='\n')?'n':'0');

+                        else if(c<' ' || c>'~' || c=='\\')

+                                fprintf(stderr,"\\x%02X",((unsigned char *)redata->chunks[m]->data)[k]);

+                        else

+                                fprintf(stderr,"%c",c);

+                }

+                fprintf(stderr,"\"\n");

+        }

+}

+#define CHUNKDEBUG(a) redata_debug_chunkdump a

+#else

+#define CHUNKDEBUG(a)

+#endif

+

 #include "re_plugin_unsaved.h"

 #include "re_plugin_highlighter.h"

 #include "re_ui.h"

@@ -416,7 +445,7 @@ re_processkey_editing(re_t *re, SDL_Event *event)

                 strncpy(event->text.text,spaces+(re->curcol%8),sizeof(event->text.text));

                 event->text.text[sizeof(event->text.text)-1]='\0';

         } else if(event->type==SDL_KEYDOWN && event->key.keysym.sym==SDLK_DELETE) {

-#if 1

+#if 0

 fprintf(stderr,"SDL_KEYDOWN: DELETE\n");

 #endif

                 long realend;

@@ -969,6 +998,10 @@ re_drawcontents(re_t *re)

         int in_error;

         long realstart,realend;

         int drawn_cursor;

+        hcolor_t *colors;

+        int ncolors;

+        linecolor_t *linecolors;

+        int nlinecolors;

         if(re==NULL)

                 return(-1);

         reui_fill(re->ui,re->x,re->y,re->w,re->h,"\xdf\xdf\xdf\xff");

@@ -985,11 +1018,35 @@ re_drawcontents(re_t *re)

         reui_fill(re->ui,re->x,re->y+(re->curline-re->originline)*re->ui->fontheight,re->w,re->ui->fontheight+1,"\xef\xef\xef\xff");

         /* draw the lines */

         drawn_cursor=0;

+        colors=redata_highlighter_getcolors(re->data,&ncolors);

+#if 0

+CHUNKDEBUG((re->data,"Pre-display"));

+#endif

         for(y=re->y;y<(re->y+re->h);y+=re->ui->fontheight,row++) {

+                /* definiciton of vars for teacking linecolor usage */

+                int curlinecolor; /* current linecolor */

+                int usedlenlinecolor; /* number of bytes of current linecolor already drawn */

+                /* end of deifinitions */

                 if(redata_line_realstart(re->data,pos,&realstart)==-1 || redata_line_realend(re->data,pos,&realend)==-1) {

                         break; /* couldn't get real start/end */

                 }

                 in_error=0;

+                linecolors=(colors==NULL)?NULL:redata_highlighter_getline(re->data,re->originline+row,&nlinecolors);

+#if 0

+#define DEBUGLINE 12

+if((re->origincol+row)==DEBUGLINE) {

+int i,k,l=nlinecolors*sizeof(linecolor_t);

+linecolor_t *lc=linecolors;

+fprintf(stderr,"linecolors for line:%i \"",re->origincol+row);

+for(k=0;k<l;k+=sizeof(linecolor_t),lc++) {

+        for(i=0;i<lc->len;i++)

+                fprintf(stderr,"%x",lc->color);

+}

+fprintf(stderr,"\"\n");

+}

+#endif

+                curlinecolor=0;

+                usedlenlinecolor=0;

                 for(tmpcol=0,pos=realstart,availcol=0;tmpcol<(re->origincol+re->maxcol) && pos<=realend;pos=newpos+len,tmpcol+=availcol) {

                         if(redata_line_rawinfo(re->data,pos,&newpos,&ptr,&len,&is_continuation)==-1) {

                                 in_error=1;

@@ -998,7 +1055,53 @@ re_drawcontents(re_t *re)

                         has_nl=((len>0 && ptr[len-1]=='\n')?1:0);

                         availcol=redata_generic_utf8len(ptr,len-has_nl);

 #warning TODO: consider tabs

-                        reui_write(re->ui,re->x+(tmpcol-re->origincol)*re->ui->fontwidth,y,"\x00\x00\x00\xff",ptr,len-has_nl);

+                        if(linecolors!=NULL) {

+                                int used,usedcol; /* number of bytes/columns used of redata chunk (those are already drawn) */