@@ -29,13 +29,28 @@

 //#define CHUNKSIZE 4096

 //#define CHUNKSIZE 1024

 //#define CHUNKSIZE 160

-#define CHUNKSIZE 16

+//#define CHUNKSIZE 16

+#define CHUNKSIZE 3

+#warning XXX: TODO: CHANGE CHUNKSIZE TO A SANE VALUE (not 1!), THIS IS ONLY FOR TESTING UTF-8 multi-byte CHARS

+

 #define UNDOBLOCK 1024

 #define ADDNBLOCK 1024

 #define UNDOGROWSIZE (256*1024)

 #define SECURESAVEPREFIX "."

 #define SECURESAVEPOSTFIX ".saving"

+#define UTF8_IS_ASCII_OR_START(c) (((c)&0xc0)!=0x80)

+#define UTF8_IS_ASCII(c) (((c)&0x80)==0)

+#define UTF8_IS_MULTIBYTESTART(c) ((((c)&0x80)==0x80)&&(((c)&(0x80|0x40))!=0x80))

+#define UTF8_IS_MULTIBYTECONT(c) (((c)&(0x80|0x40))==0x80)

+

+#define UTF8_MULTIBYTESTART2LEN(c) ((((c)&0xe0)==0xc0)?2: \

+                                    (((c)&0xf0)==0xe0)?3: \

+                                    (((c)&0xf8)==0xf0)?4: \

+                                    (((c)&0xfc)==0xf8)?5: \

+                                    (((c)&0xfe)==0xfc)?6: \

+                                    1)

+

 static int redata_hash_gen(redata_t *redata, char *filename, char *buf, long buflen, char *resbuf129bytes);

 static char *securesave_genname(char *filename, char *buf, int bufsize);

 static void *mymemrchr(const void *s, int c, size_t n);

@@ -1587,6 +1602,50 @@ redata_memcmp(redata_t *redata, long pos, char *str, int len)

 }

+int

+redata_getutf8char(redata_t *redata, long pos, char *buf, int len, int *usedbuf)

+{

+        int numchunk;

+        int offset;

+        rechunk_t *chunk;

+        int ooff;

+        int req;

+        int c;

+        if(redata==NULL || pos<0 || pos>redata_getsize(redata) || buf==NULL || len<1 || usedbuf==NULL)

+                return(-1); /* sanity check failed */

+        ooff=0;

+        if(redata_getposptr(redata,pos,&numchunk,&offset)!=0)

+                return(-1); /* couldn't get pos */

+        while(numchunk<redata->sizechunks && redata->chunks[numchunk]!=NULL && offset>=redata->chunks[numchunk]->useddata) {

+                numchunk++;

+                offset=0;

+        }

+        if(numchunk>=redata->sizechunks)

+                return(-1); /* at end; no data */

+        chunk=redata->chunks[numchunk];

+        c=chunk->data[offset];

+        ((unsigned char *)buf)[ooff++]=c;

+        if(UTF8_IS_ASCII(c) || UTF8_IS_MULTIBYTECONT(c))

+                req=1;

+        else

+                req=UTF8_MULTIBYTESTART2LEN(c);

+        offset++;

+        for(;ooff<len && ooff<req && numchunk<redata->sizechunks

+            ;numchunk++,chunk=redata->chunks[numchunk],offset=0) {

+                if(offset>=chunk->useddata)

+                        continue;

+                c=0x80|0x40; /* initialize c to whatever multibycont for the check after the while */

+                while(ooff<len && ooff<req

+                  && (c=chunk->data[offset])!=0 && UTF8_IS_MULTIBYTECONT(c)) {

+                        ((unsigned char *)buf)[ooff++]=c;

+                }

+                if(!UTF8_IS_MULTIBYTECONT(c))

+                        break;

+        }

+        *usedbuf=ooff;

+        return(0);

+}

+

 int

 redata_generic_utf8len(char *ptr, int size)

@@ -1606,6 +1665,63 @@ redata_generic_utf8len(char *ptr, int size)

 /*#warning TODO: XXX Also consider tabs*/

 }

+int

+redata_generic_utf8lenincomplete(char *ptr, int size, int *nstartincomplete,int *nendincomplete, int *nendrequired)

+{

+        int len,i;

+        int lastistart,lastclen;

+/*#warning TODO: XXX support combining code points (at least U+0300 - U+036F ( https://en.wikipedia.org/wiki/Combining_character ) */

+        if(size<0 || (ptr==NULL && size!=0) || nstartincomplete==NULL || nendincomplete==NULL)

+                return(-1);

+        /* from RFC2279/RFC3629, one character is up to 6 bytes:

+           first    last     Byte1      Byte2      Byte3      Byte4

+           U+0000   U+007F   0xxxxxxx

+           U+0080   U+07FF   110xxxxx   10xxxxxx

+           U+0800   U+FFFF   1110xxxx   10xxxxxx   10xxxxxx

+          U+10000 U+10FFFF   11110xxx   10xxxxxx   10xxxxxx   10xxxxxx

+            ...

+         */   

+        i=0;

+        len=0;

+        /* nstartincomplete */

+        *nstartincomplete=0;

+        while(i<size && (ptr[i]&(0x80|0x40))==0x80) {

+                (*nstartincomplete)++;

+                i++;

+        }

+        /* len */

+        lastistart=-1;

+        lastclen=0;

+        for(;i<size;i++) { 

+                len+=((ptr[i]&0xc0)!=0x80)?1:0;

+                if((ptr[i]&0x80)==0x00) {

+                        /* ASCII */

+                        lastistart=i;

+                        lastclen=1;

+                } else if((ptr[i]&0xc0)!=0x80) {

+                        /* multibytechar start (as ASCII case has already been processed) */

+                        lastistart=i;

+                        lastclen=((ptr[i]&0xe0)==0xc0)?2:

+                                 ((ptr[i]&0xf0)==0xe0)?3:

+                                 ((ptr[i]&0xf8)==0xf0)?4:

+                                 ((ptr[i]&0xfc)==0xf8)?5:

+                                 ((ptr[i]&0xfe)==0xfc)?6:

+                                 1; /* unknown type of multibytechar */

+                }

+        }

+        /* nendincomplete */

+        *nendincomplete=0;

+        *nendrequired=0;

+        if(lastistart!=-1 && (lastistart+lastclen)>size) {

+                *nendrequired=lastclen;

+                *nendincomplete=(lastistart+lastclen-size);

+                len--;

+        }

+        /* all done */

+        return(len);

+/*#warning TODO: XXX Also consider tabs*/

+}

+

 char *

 redata_generic_utf8col(char *ptr, int size, int col)

 {

@@ -1654,7 +1770,6 @@ redata_generic_utf8isstartbyte(int candidate)

         return(0);

 }

-

 static int

 redata_hash_gen(redata_t *redata, char *filename, char *buf, long buflen, char *resbuf129bytes)

 {

@@ -1933,7 +2048,7 @@ redata_linecol2pos(redata_t *redata, int line, int colrequest, long *pos, int *c

         int i,n;

         char *ptr;

         int len;

-        int done;

+        int done;                       

         if(redata==NULL || line<0 || colrequest<0)

                 return(-1);

         /* find line */

@@ -1958,21 +2073,18 @@ redata_linecol2pos(redata_t *redata, int line, int colrequest, long *pos, int *c

                 return(-1); /* line not found */

         realstart=chunkpos+i;

         /* find col */

-        for(n=0,curpos=realstart;n<colrequest;) {

+        for(n=-1,curpos=realstart;n<colrequest;) {

                 if(redata_line_rawinfo(redata,curpos,&startpos,&ptr,&len,NULL)==-1 || len==0)

                         return(-1); /* couldn't get current line data */

                 done=0;

-                for(i=0;colrequest>0 && i<len && n<=colrequest;i++) {

-                        if(redata_generic_utf8isstartbyte(ptr[i])) {

-                                if(n==colrequest) {

-                                        done=1;

-                                        break;

-                                }

-                                n++;

-                        }

+                for(i=0;i<len && n<colrequest;i++) {

                         if(ptr[i]=='\n') {

                                 done=1;

                                 break;

+                        } else if(UTF8_IS_ASCII_OR_START(ptr[i])) {                    

+                                n++;

+                                if(n==colrequest)

+                                        break;

                         }

                 }

                 curpos=startpos+i;

@@ -171,10 +171,17 @@ long redata_searchforward(redata_t *redata, long posini, char *str, int len);

 long redata_searchbackwards(redata_t *redata, long posini, char *str, int len);

 int redata_memcmp(redata_t *redata, long pos, char *str, int len);

+int redata_getutf8char(redata_t *redata, long pos, char *buf, int len, int *usedbuf);

+

 /* utf8 convenience functions */

 /* calculate the number of utf8-charaters in buffer */

 int redata_generic_utf8len(char *ptr, int size);

+

+/* calculate the number of utf8-charaters in buffer ignoring incomplete multibytechars at start and end */

+int redata_generic_utf8lenincomplete(char *ptr, int size, int *nstartincomplete,int *nendincomplete, int *nendrequired

+);

+

 /* return a pointer to the "n"th ("col"th) utf8-character in buffer */

 char *redata_generic_utf8col(char *ptr, int size, int col);

 /* returns the len in bytes of the character starting at ptr (zero on error)*/

@@ -1889,31 +1889,24 @@ re_drawcontents(re_t *re, printout_t *printout)

         int curline,curcol;

         int maxcol,maxrow;

         long pos,newpos;

-        char *ptr;

-        int len;

         int y,row,tmprow;

-        char *curptr;

-        int curptrlen;

-        int has_nl;

+        long cursorpos;

         int is_continuation;

-        int tmpcol,availcol;

-        int in_error;

-        long realstart,realend;

-        int drawn_cursor;

-        hcolor_t *colors;

+        printout_t fakeprintout;

+        hcolor_t *colors,fakecolor;

         int ncolors;

-        linecolor_t *linecolors;

+        const char selcolornormal[]={"\xde\xcf\x7f\xff"};

+        const char selcolorcurline[]={"\xf0\xea\xc9\xff"};

+        long realstart,realend;

+        linecolor_t *linecolors,fakelinecolors;

         int nlinecolors;

+        int drawn_cursor;

         int matchingpos;

         char matchingchar;

         int mline,mcol;

-        long cursorpos;

-        printout_t fakeprintout;

-        const char *hint;

-        const char selcolornormal[]={"\xde\xcf\x7f\xff"};

-        const char selcolorcurline[]={"\xf0\xea\xc9\xff"};

         if(re==NULL || (printout!=NULL && (printout->ui==NULL || printout->data==NULL)))

                 return(-1);

+        /* init vars and clear screen */

         ui=re->ui;

         data=re->data;

         x0=re->x;

@@ -1950,6 +1943,7 @@ re_drawcontents(re_t *re, printout_t *printout)

         reui_fill(ui,x0,y0,w,h,"\xdf\xdf\xdf\xff");

         row=curline-originline;

         pos=cursorpos;

+        /* get position/row/col of character at top left of screen */

         while(row>0 && pos>0) {

                 if(redata_line_rawinfo(data,pos,&newpos,NULL,NULL,&is_continuation)==-1)

                         return(-1);

@@ -1985,70 +1979,108 @@ re_drawcontents(re_t *re, printout_t *printout)

                                 if(x2>(x0))

                                         reui_fill(ui,x0,y0+row*ui->fontheight,x2-x0,ui->fontheight+1,selcolor);

                         }

-

                 }

                 row=tmprow;

         }

         /* draw the lines */

+        if((colors=redata_highlighter_getcolors(data,&ncolors))==NULL) {

+                colors=&fakecolor;

+                ncolors=1;

+                memcpy(fakecolor.rgba,"\x00\x00\x00\xff",5);

+        }

         drawn_cursor=0;

-        colors=redata_highlighter_getcolors(data,&ncolors);

         matchingpos=-1;

         matchingchar='\0';

         for(y=y0;y<(y0+h);y+=ui->fontheight,row++) {

+                int in_error;

+                int availcol,tmpcol,len;

+                char *lastcolor;

                 /* definition of vars for tracking linecolor usage */

                 int curlinecolor; /* current linecolor */

                 int usedlenlinecolor; /* number of bytes of current linecolor already drawn */

-                /* end of definitions */

-                if(redata_line_realstart(data,pos,&realstart)==-1 || redata_line_realend(data,pos,&realend)==-1) {

+                /* get start/end of line */

+                if(redata_line_realstart(data,pos,&realstart)==-1 || redata_line_realend(data,pos,&realend)==-1)

                         break; /* couldn't get real start/end */

+                /* setup colors */

+                if(colors==(&fakecolor) || (linecolors=redata_highlighter_getline(data,originline+row,&nlinecolors))==NULL) {

+                        linecolors=&fakelinecolors;

+                        fakelinecolors.len=1;

+                        fakelinecolors.color=0;

                 }

-                in_error=0;

-                linecolors=(colors==NULL)?NULL:redata_highlighter_getline(data,originline+row,&nlinecolors);

                 curlinecolor=0;

                 usedlenlinecolor=0;

+                lastcolor="\x00\x00\x00\xff";                      

+                in_error=0;

+                /* draw each part of this line */

                 for(tmpcol=0,pos=realstart,availcol=0;tmpcol<(origincol+maxcol) && pos<=realend;pos=newpos+len,tmpcol+=availcol) {

+                        int has_nl;

+                        char *ptr;

+                        int incompletestart,incompleteend,endreq;

+                        int used,usedcol; /* number of bytes/columns used of redata chunk (those are already drawn) */

                         if(redata_line_rawinfo(data,pos,&newpos,&ptr,&len,&is_continuation)==-1) {

                                 in_error=1;

                                 break; /* couldn't get this line part info */

                         }

                         has_nl=((len>0 && ptr[len-1]=='\n')?1:0);

-                        availcol=redata_generic_utf8len(ptr,len-has_nl);

-#warning TODO: consider tabs

-                        if(linecolors!=NULL) {

-                                int used,usedcol; /* number of bytes/columns used of redata chunk (those are already drawn) */

-                                used=usedcol=0;

-                                /* while the avail text is larger than the linecolor len */

-                                while(curlinecolor<nlinecolors && (len-has_nl-used)>=(linecolors[curlinecolor].len-usedlenlinecolor)) {

-                                        reui_write(ui,x0+(tmpcol-origincol+usedcol)*ui->fontwidth,y,colors[linecolors[curlinecolor].color].rgba,ptr+used,linecolors[curlinecolor].len-usedlenlinecolor);

-                                        usedcol+=redata_generic_utf8len(ptr+used,linecolors[curlinecolor].len-usedlenlinecolor);

-                                        used+=linecolors[curlinecolor].len-usedlenlinecolor;

-                                        curlinecolor++;

-                                        usedlenlinecolor=0;

-                                }

-                                /* for the last bytes of avail text, after writing them we save how many bytes we have processed of that linecolor to be able to continue later */

-                                if(curlinecolor<nlinecolors && (len-has_nl-used)>0 && (linecolors[curlinecolor].len-usedlenlinecolor)>(len-has_nl-used)) {

-                                        reui_write(ui,x0+(tmpcol-origincol+usedcol)*ui->fontwidth,y,colors[linecolors[curlinecolor].color].rgba,ptr+used,(len-has_nl-used));

-                                        usedcol+=redata_generic_utf8len(ptr+used,(len-has_nl-used));

-                                        usedlenlinecolor+=(len-has_nl-used);

-                                        used+=(len-has_nl-used);

+                        availcol=redata_generic_utf8lenincomplete(ptr,len-has_nl,&incompletestart,&incompleteend,&endreq);

+                        /* display the line */

+                        used=incompletestart;

+                        usedcol=0;

+                        /* while the avail text is larger than the linecolor len */

+                        while(curlinecolor<nlinecolors && (len-has_nl-incompleteend-used)>=(linecolors[curlinecolor].len-usedlenlinecolor)) {

+                                lastcolor=colors[linecolors[curlinecolor].color].rgba;

+                                reui_write(ui,x0+(tmpcol-origincol+usedcol)*ui->fontwidth,y,lastcolor,ptr+used,linecolors[curlinecolor].len-usedlenlinecolor);

+                                usedcol+=redata_generic_utf8len(ptr+used,linecolors[curlinecolor].len-usedlenlinecolor);

+                                used+=linecolors[curlinecolor].len-usedlenlinecolor;

+                                curlinecolor++;

+                                usedlenlinecolor=0;

+                        }

+                        /* for the last bytes of avail text, after writing them we save how many bytes we have processed of that linecolor to be able to continue later */

+                        if((len-has_nl-incompleteend-used)>0) {

+                                if(curlinecolor<nlinecolors && (linecolors[curlinecolor].len-usedlenlinecolor)>(len-has_nl-incompleteend-used)) {

+                                        lastcolor=colors[linecolors[curlinecolor].color].rgba;

+                                        usedlenlinecolor+=(len-has_nl-incompleteend-used);

+                                        if(usedlenlinecolor==linecolors[curlinecolor].len) {

+                                                curlinecolor++;

+                                                usedlenlinecolor=0;

+                                        }

                                 }

-                        } else {

-                                reui_write(ui,x0+(tmpcol-origincol)*ui->fontwidth,y,"\x00\x00\x00\xff",ptr,len-has_nl);

+                                reui_write(ui,x0+(tmpcol-origincol+usedcol)*ui->fontwidth,y,lastcolor,ptr+used,(len-has_nl-incompleteend-used));

+                                usedcol+=redata_generic_utf8len(ptr+used,(len-has_nl-incompleteend-used));

+                                used+=(len-has_nl-incompleteend-used);

                         }

-                        if(printout==NULL && row==(curline-originline)) {

-                                if(curcol>=tmpcol && curcol<(tmpcol+availcol)) {

-                                        int utf8charlen;

-                                        /* draw cursor */

-                                        reui_fill(ui,x0+ui->fontwidth*(curcol-origincol),y,ui->fontwidth,ui->fontheight+1,"\x00\x00\x00\xff");

-                                        drawn_cursor=1;

-                                        curptr=redata_generic_utf8col(ptr,len-has_nl,curcol-tmpcol);

-                                        curptrlen=(curptr==NULL)?0:(len-has_nl)-(curptr-ptr);

-                                        utf8charlen=redata_generic_utf8charlen(curptr,curptrlen);

-                                        reui_write(ui,x0+ui->fontwidth*(curcol-origincol),y,"\xff\xff\xff\xff",curptr,utf8charlen);

-                                        /* get matching braces/parens/anglebracket/curlybraces for highlighting */

-                                        if(utf8charlen==1 && strchr("[]{}<>()",*curptr)!=NULL)

-                                                matchingpos=re_getmatchingbracket(re,cursorpos,*curptr,&matchingchar);

+                        /* if the last utf-8 char is broken into several blocks */

+                        if(incompleteend>0) {

+                                char broken[7];

+                                int usedbroken;

+                                usedbroken=0;

+                                redata_getutf8char(re->data,newpos+len-has_nl-incompleteend,broken,sizeof(broken),&usedbroken);

+                                /* write the just-recomposer character in the correct color */

+                                if(curlinecolor<nlinecolors && (linecolors[curlinecolor].len-usedlenlinecolor)>=1) {

+                                        lastcolor=colors[linecolors[curlinecolor].color].rgba;

+                                        usedlenlinecolor+=1;

+                                        if(usedlenlinecolor==linecolors[curlinecolor].len) {

+                                                curlinecolor++;

+                                                usedlenlinecolor=0;

+                                        }

                                 }

+                                reui_write(ui,x0+(tmpcol-origincol+usedcol)*ui->fontwidth,y,lastcolor,broken,usedbroken);

+                                usedcol++;

+                                availcol++;

+                        }

+                        /* draw cursor if applicable */

+                        if(printout==NULL && row==(curline-originline) && curcol>=tmpcol && curcol<(tmpcol+availcol)) {

+                                char cursorchar[7];

+                                int usedcursorchar;

+                                /* draw cursor */

+                                reui_fill(ui,x0+ui->fontwidth*(curcol-origincol),y,ui->fontwidth,ui->fontheight+1,"\x00\x00\x00\xff");

+                                drawn_cursor=1;

+                                usedcursorchar=0;

+                                redata_getutf8char(re->data,cursorpos,cursorchar,sizeof(cursorchar),&usedcursorchar);

+                                reui_write(ui,x0+ui->fontwidth*(curcol-origincol),y,"\xff\xff\xff\xff",cursorchar,usedcursorchar);

+                                /* get matching braces/parens/anglebracket/curlybraces for highlighting */

+                                if(usedcursorchar==1 && strchr("[]{}<>()",*cursorchar)!=NULL)

+                                        matchingpos=re_getmatchingbracket(re,cursorpos,*cursorchar,&matchingchar);

                         }

                 }

                 if(printout==NULL && row==(curline-originline) && !drawn_cursor)

@@ -2056,7 +2088,6 @@ re_drawcontents(re_t *re, printout_t *printout)

                 if(in_error)

                         break;

                 pos=realend+1;

-/*LONG LINE LEFT FOR DEBUGGING PURPOSES: reui_write(ui,x0+ui->fontwidth*(curcol-origincol),y,"\xff\xff\xff\xff",curptr,redata_generic_utf8charlen(ptr,curptrlen));*/

         }

         /* highlight matching parens/brace/... if applicable */

         if(printout==NULL && matchingpos!=-1 && redata_pos2linecol(data,matchingpos,&mline,&mcol)!=-1) {

@@ -2078,14 +2109,14 @@ re_drawcontents(re_t *re, printout_t *printout)

                 else

                         reui_balloon(ui, '\0', x,y, fg, bg, &matchingchar,1);

         }

-        /* display prototypes info if applicable */

         if(printout==NULL) {

+                const char *hint;

                 hint=redata_prototypes_get(data, cursorpos);

                 if(hint!=NULL) {

-                        if((curline-originline)>=(maxrow/2))

+                        if((curline-originline)>=3)

                                 reui_balloon(ui, '\0', x0+w/2, y0+ui->fontheight*3/2, "\x80\x80\x80\xff", "\xff\xff\xff\xcf",(char *) hint,strlen(hint));

                         else

-                                reui_balloon(ui, '\0', x0+w/2, y0+h-ui->fontheight*3/2, "\x80\x80\x80\xff", "\xff\xff\xff\xcf",(char *)hint,strlen(hint));

+                                reui_balloon(ui, '\0', x0+w/2, y0+ui->fontheight*((curline-originline)*2+5)/2, "\x80\x80\x80\xff", "\xff\xff\xff\xcf",(char *) hint,strlen(hint));

                 }

         }

         /* all done */