@@ -2,7 +2,7 @@ CC=gcc

 CFLAGS=-g -Wall -Wformat-truncation=0 -Ideps -I/usr/include/SDL2

 LDFLAGS=-lSDL2 -lSDL2_image -lSDL2_ttf

-all: re tests tests_ui

+all: re tests tests_utf8

 recenteditor.o: recenteditor.c re_data.h ext/socklib.h

@@ -14,7 +14,7 @@ re_ui.o: re_ui.c re_ui.h hack_regular.h

 re_tests.o: re_tests.c re_data.h

-re_tests_ui.o: re_tests_ui.c re_ui.h

+re_tests_utf8.o: re_tests_utf8.c re_ui.h

 sha3.o: sha3/sha3.c sha3/sha3.h

 	$(CC) $(CFLAGS) -Isha3 -c -o sha3.o sha3/sha3.c

@@ -37,8 +37,8 @@ re: recenteditor.o re_data.o re_plugin_unsaved.o sha3.o re_ui.o hack_regular.o s

 tests: re_tests.o re_data.o sha3.o

 	$(CC) $(LDFLAGS) -o tests re_tests.o re_data.o sha3.o

-tests_ui: re_tests_ui.o re_ui.o hack_regular.o

-	$(CC) $(LDFLAGS) -o tests_ui re_tests_ui.o re_ui.o hack_regular.o

+tests_utf8: re_tests_utf8.o re_data.o sha3.o

+	$(CC) $(LDFLAGS) -o tests_utf8 re_tests_utf8.o re_data.o sha3.o

 clean:

 	rm -f recenteditor.o re_data.o re_tests.o re_plugin_unsaved.o sha3.o re_ui.o hack_regular.c hack_regular.h hack_regular.o re tests 

@@ -26,6 +26,7 @@

 //#define CHUNKSIZE 32768

 #define CHUNKSIZE 160

 #define UNDOBLOCK 1024

+#define ADDNBLOCK 1024

 #define UNDOGROWSIZE (256*1024)

 #define SECURESAVEPREFIX "."

 #define SECURESAVEPOSTFIX ".saving"

@@ -860,6 +861,28 @@ redata_op_add(redata_t *redata, long insertpos, char *buf, long buflen, undostac

         return(0);

 }

+int

+redata_op_addn(redata_t *redata, long pos, char character, long n, undostack_t *fromhere)

+{

+        if(redata==NULL || n<0)

+                return(-1); /* sanity check failed */

+        if(n==0)

+                return(0); /* nothing to do */

+        if(redata->sizeaddnbuf<n) {

+                char *newptr;

+                long newsize;

+                newsize=(n+1+ADDNBLOCK)/ADDNBLOCK;

+                newsize*=ADDNBLOCK;

+                if((newptr=realloc(redata->addnbuf,newsize))==NULL)

+                        return(-1); /* insuf. mem. */

+                redata->addnbuf=newptr;

+                redata->sizeaddnbuf=newsize;

+        }

+        memset(redata->addnbuf,character,n);

+        redata->addnbuf[n]='\0';

+        return(redata_op_add(redata, pos, redata->addnbuf, n, fromhere));

+}

+

 int

 redata_op_del(redata_t *redata, long delpos, long size, undostack_t *fromhere)

 {

@@ -1249,6 +1272,64 @@ redata_generic_genname(char *filename,char *prefix, char *postfix, char *buf, in

         return(name);

 }

+int

+redata_generic_utf8len(char *ptr, int size)

+{

+        int len,i;

+        /* calculate the number of utf8-charaters in buffer */

+        if(size<0 || (ptr==NULL && size!=0))

+                return(-1);

+        /* for now we only count the number of code points */

+        /* in UTF8: 0x00-0x7f single byte chars

+         *          0xc0-0xff leading bytes

+         *          0x80-0xbf continuation bytes (ignore these for len)*/

+/*#warning TODO: XXX support combining code points (at least U+0300 - U+036F ( https://en.wikipedia.org/wiki/Combining_character ) */

+        for(len=0,i=0;i<size;i++)

+                len+=((ptr[i]&0xc0)!=0x80)?1:0;

+        return(len);

+/*#warning TODO: XXX Also consider tabs*/

+}

+

+char *

+redata_generic_utf8col(char *ptr, int size, int col)

+{

+        int len,i;

+        /* return a pointer to the "n"th ("col"th) utf8-character in buffer */

+        if(size<0 || (ptr==NULL && size!=0))

+                return(NULL); /* sanity check failed */

+        /* see reui_utf8len() for explanation of algorithm */

+/*#warning TODO: support combining code points (at least U+0300 - U+036F ( https://en.wikipedia.org/wiki/Combining_character ) */

+        if(col>=size)

+                return(NULL);/* col greater than maximum possible char. count */

+        /* skip "col" amount of single byte chars and leading bytes */

+        for(len=0,i=0;len<col && i<size;i++)

+                len+=((ptr[i]&0xc0)!=0x80)?1:0;

+        /* if we landed in a continuation byte, advance until next single byte chars or leading byte */

+        while(i<size && (ptr[i]&0xc0)==0x80)

+                i++;

+        if(i>=size)

+                return(NULL); /* col is beyond end of string */

+        return(ptr+i);

+/*#warning TODO: XXX Also consider tabs*/

+}

+

+int

+redata_generic_utf8charlen(char *ptr, int maxsize)

+{

+        int i;

+        /* returns the len in bytes of the character starting at ptr (zero on error)*/

+        if(ptr==NULL || maxsize<1)

+                return(0); /* sanity check failed */

+/*#warning TODO: support combining code points (at least U+0300 - U+036F ( https://en.wikipedia.org/wiki/Combining_character ) */

+        if(((unsigned char *)ptr)[0]<0x80)

+                return(1); /* single byte char */

+        if((ptr[0]&0xc0)==0x80)

+                return(0); /* error: this is continuation, not leading byte */

+        for(i=1;i<maxsize && (ptr[i]&0xc0)==0x80;i++)

+                ;

+        return(i);

+}

+

 static int

 redata_hash_gen(redata_t *redata, char *filename, char *buf, long buflen, char *resbuf129bytes)

@@ -68,6 +68,8 @@ typedef struct redata_t {

         rechunk_t **chunks;

         long available;

         rechunk_t *tmpchunk;

+        long sizeaddnbuf;

+        char *addnbuf;

         /* undo */

         undostack_t undostack;

         undostack_t redostack;

@@ -113,6 +115,7 @@ int redata_load(redata_t *redata, char *filename, int (*callback_question)(/*cha

 int redata_save(redata_t *redata, char *filename);

 int redata_op_add(redata_t *redata, long pos, char *buf, long buflen, undostack_t *fromhere);

+int redata_op_addn(redata_t *redata, long pos, char character, long n, undostack_t *fromhere);

 int redata_op_del(redata_t *redata, long pos, long size, undostack_t *fromhere);

 int redata_op_move(redata_t *redata, long posorig, long size, long posdest, undostack_t *fromhere);

 int redata_op_undo(redata_t *redata);

@@ -126,7 +129,14 @@ int redata_hash(redata_t *redata, char *resbuf129bytes);

 int redata_filehash(redata_t *redata, char *filename, char *resbuf129bytes);

 int redata_memhash(redata_t *redata, char *buf, long buflen, char *resbuf129bytes);

 undostack_t *redata_getstack(redata_t *redata, undo_t *undo);

+

 char *redata_generic_genname(char *filename,char *prefix, char *postfix, char *buf, int bufsize);

+int redata_generic_utf8len(char *ptr, int size);

+char *redata_generic_utf8col(char *ptr, int size, int col);

+int redata_generic_utf8charlen(char *ptr, int maxsize);

+

+

 /* line convenience funtions */

 int redata_line_info(redata_t *redata, long pos, long *startpos, char **startptr, int *len);

+

deleted file mode 100644

@@ -1,150 +0,0 @@

-/*

- * re_tests_ui.c

- *

- * A programmers editor

- *

- * Tests (ensure correct functionality of modules)

- *

- * Author: Dario Rodriguez dario@softhome.net

- * This program is licensed under the terms of GNU GPL v2.1+

- */

-

-#include <stdio.h>

-#include <stdlib.h>

-#include <unistd.h>

-#include <string.h>

-#include <limits.h>

-#include <fcntl.h>

-#include <errno.h>

-

-#include "re_ui.h"

-

-#define TEST_OK "OK"

-

-typedef struct test_ui_t  {

-        int needs_realui;

-        char *name;

-        char *(*fn)(reui_t *,char *,int , int, int, int);

-} test_ui_t;

-

-char *test_utf8len(reui_t *ui, char *teststring, int expectednchars, int dummy1, int dummy2,int dummy3);

-char *test_utf8col(reui_t *ui, char *teststring, int dummy0, int col, int expectedoffset,int dummy3);

-char *test_utf8charlen(reui_t *ui, char *teststring, int dummy0, int dummy1, int offset,int charsizeatoffset);

-

-int

-main(int argc, char *argv[])

-{

-        struct reui_t dummyui;

-        struct {

-                char string[1024];

-                int nchars;

-                int col;

-                int offsetcol;

-                int charsizeatoffsetcol;

-        } teststrings[]={

-                {{"This is a latin1 string"},51-28,1,1,1},

-                {{"lowercase acute vowels:\xc3\xa1\xc3\xa9\xc3\xad\xc3\xb3\xc3\xba"},23+5,23+1,23+2,2},

-        };

-        test_ui_t tests[]={

-                {0,"utf8len",test_utf8len},

-                {0,"utf8col",test_utf8col},

-                {0,"utf8charlen",test_utf8charlen},

-        };

-        int flag_exit=0,flag_all=0;

-        reui_t *ui;

-        int i,s;

-        int nerrors,total;

-        char *res;

-        for(i=1;i<argc;i++) {

-                if(strcmp(argv[i],"--help")==0) {

-                        fprintf(stderr,"Syntax: %s [--all] [--exit] [--help]\nExplanation:\n\t--all: do even the slow tests\n\t--exit: exit program at first unsuccessful test\n\t--help: this text\n",argv[0]);

-                        return(1);

-                } else if(strcmp(argv[i],"--all")==0) {

-                        flag_all=1;

-                } else if(strcmp(argv[i],"--exit")==0) {

-                        flag_exit=1;

-                }

-        }

-        nerrors=0;

-        total=0;

-        /* flag_all is not used right now, next line is to silence the compiler */

-        total+=(flag_all&0);

-        /* end of hack */

-        memset(&dummyui,0,sizeof(reui_t));

-        for(s=0;s<(sizeof(teststrings)/sizeof(teststrings[0]));s++) {

-                fprintf(stderr,"\"%s\"\n",teststrings[s].string);

-                for(i=0;i<(sizeof(tests)/sizeof(tests[0]));i++,total++) {

-                        ui=&dummyui;

-                        if(tests[i].needs_realui && (ui=reui_init(NULL))==NULL) {

-                                fprintf(stderr,"ERROR: problem initializing ui module\n");

-                                return(1);

-                        }

-                        fprintf(stderr,"%i:%s...",total+1,tests[i].name);

-                        res=tests[i].fn(ui,teststrings[s].string,teststrings[s].nchars,teststrings[s].col,teststrings[s].offsetcol,teststrings[s].charsizeatoffsetcol);

-                                if(strcmp(res,TEST_OK)==0) {

-                                fprintf(stderr," ok.\n");

-                        } else {

-                                fprintf(stderr," ERROR: %s <= %s(\"%s\",%i,%i,%i,%i)\n",res,tests[i].name,teststrings[s].string,teststrings[s].nchars,teststrings[s].col,teststrings[s].offsetcol,teststrings[s].charsizeatoffsetcol);

-                                nerrors++;

-                                if(flag_exit) {

-                                        /* exit on first error */

-                                        s=sizeof(teststrings)/sizeof(teststrings[0]);

-                                        break;

-                                }

-                        }

-                        if(tests[i].needs_realui)

-                                reui_free(ui),ui=NULL;

-                }

-        }

-        fprintf(stderr,"\n");

-        if(nerrors==0)

-                fprintf(stderr,"All %i tests passed OK\n",total);

-        else

-                fprintf(stderr,"%i test(s) failed of %i tests run.\n",nerrors,total);

-        if(ui!=NULL && ui!=&dummyui)

-                reui_free(ui),ui=NULL;

-        return((nerrors==0)?0:1);

-}

-

-char *

-test_utf8len(reui_t *ui, char *teststring, int expectednchars, int dummy1, int dummy2, int dummy3)

-{

-        int res;

-        static char errorstr[1024];

-        res=reui_utf8len(ui,teststring,strlen(teststring));

-        if(res!=expectednchars) {

-                snprintf(errorstr,sizeof(errorstr),"expected %i chars, got %i chars",expectednchars,res);

-                errorstr[sizeof(errorstr)-1]='\0';

-                return(errorstr);

-        }

-        return(TEST_OK);

-}

-

-char *test_utf8col(reui_t *ui, char *teststring, int dummy0, int col, int expectedoffset, int dummy3)

-{

-        char *ptr;

-        static char errorstr[1024];

-        ptr=reui_utf8col(ui,teststring,strlen(teststring),col);

-        if(ptr!=(teststring+expectedoffset)) {

-                snprintf(errorstr,sizeof(errorstr),"expected offset %i, got offset %i (\"%s\")",expectedoffset,(int) ((ptr==NULL)?-1:ptr-teststring),ptr);

-                errorstr[sizeof(errorstr)-1]='\0';

-                return(errorstr);

-        }

-        return(TEST_OK);

-}

-

-char *

-test_utf8charlen(reui_t *ui, char *teststring, int dummy0, int dummy1, int offset,int charsizeatoffset)

-{

-        int res;

-        static char errorstr[1024];

-        res=reui_utf8charlen(ui,teststring+offset,strlen(teststring+offset));

-        if(res!=(charsizeatoffset)) {

-                snprintf(errorstr,sizeof(errorstr),"expected char size %i, got char size %i (\"%s\")",charsizeatoffset,res,teststring+offset);

-                errorstr[sizeof(errorstr)-1]='\0';

-                return(errorstr);

-        }

-        return(TEST_OK);

-}

-

-

@@ -219,11 +219,11 @@ re_processkey(re_t *re, SDL_Event *event)

                         return(-1); /* couldn't get current line data */

                 if(event->key.keysym.sym==SDLK_UP && newpos==0)

                         return(-1); /* going up but already at top */

-                oldcol=reui_utf8len(re->ui,ptr,re->cursorpos-newpos);

+                oldcol=redata_generic_utf8len(ptr,re->cursorpos-newpos);

                 if(redata_line_info(re->data,(event->key.keysym.sym==SDLK_DOWN)?(newpos+len):(newpos-1),&newpos2,&ptr,&len)==-1)

                         return(-1); /* couldn't get next line data */

                 has_nl=((len>0 && ptr[len-1]=='\n')?1:0);

-                ptr2=reui_utf8col(re->ui,ptr,len-has_nl,oldcol);

+                ptr2=redata_generic_utf8col(ptr,len-has_nl,oldcol);

                 if(ptr2!=NULL)

                         re->cursorpos=newpos2+(ptr2-ptr);

                 else

@@ -239,12 +239,12 @@ re_processkey(re_t *re, SDL_Event *event)

                         return(-1); /* couldn't get current line data */

                 if(event->key.keysym.sym==SDLK_LEFT && re->cursorpos==0)

                         return(-1); /* going left but already at leftmost char */

-                linecols=reui_utf8len(re->ui,ptr,len);

-                oldcol=reui_utf8len(re->ui,ptr,re->cursorpos-newpos);

+                linecols=redata_generic_utf8len(ptr,len);

+                oldcol=redata_generic_utf8len(ptr,re->cursorpos-newpos);

                 inc=(event->key.keysym.sym==SDLK_LEFT)?-1:1;

                 has_nl=((len>0 && ptr[len-1]=='\n')?1:0);

                 if(re->lastcol<=linecols) {

-                        ptr2=reui_utf8col(re->ui,ptr,len,re->lastcol+inc);

+                        ptr2=redata_generic_utf8col(ptr,len,re->lastcol+inc);

                 } else

                         ptr2=NULL;

                 if(ptr2!=NULL)

@@ -255,6 +255,8 @@ re_processkey(re_t *re, SDL_Event *event)

                         re->lastcol+=inc;

                 re_drawheader(re);

                 re->contentsdirty=1;

+        } else if(event->key.keysym.sym==SDLK_a) {

+                ;

         }

         return(-1);

 }

@@ -300,12 +302,12 @@ re_drawcontents(re_t *re)

                 has_nl=((len>0 && ptr[len-1]=='\n')?1:0);

                 reui_write(re->ui,re->x,y,"\x00\x00\x00\xff",(char *)ptr,len-has_nl);

                 if(row==re->lastrow) {

-     #warning DEBUG write of current char

+#warning DEBUG write of current char

                         reui_fill(re->ui,re->x+re->ui->fontwidth*re->lastcol,y,re->ui->fontwidth,re->ui->fontheight+1,"\x00\x00\x00\xff");

 #warning TODO: consider tabs

-                        curptr=reui_utf8col(re->ui,ptr,len-has_nl,re->lastcol);

+                        curptr=redata_generic_utf8col(ptr,len-has_nl,re->lastcol);

                         curptrlen=(curptr==NULL)?0:(len-has_nl)-(curptr-ptr);

-                        reui_write(re->ui,re->x+re->ui->fontwidth*re->lastcol,y,"\xff\xff\xff\xff",curptr,reui_utf8charlen(re->ui,ptr,curptrlen));

+                        reui_write(re->ui,re->x+re->ui->fontwidth*re->lastcol,y,"\xff\xff\xff\xff",curptr,redata_generic_utf8charlen(ptr,curptrlen));

 #warning TODO: if it is one of  '[','{','<','>','}',']', highlight the matching bracket/parens/anglebracket.

                 }

                 pos=newpos+len;