To: vim_dev@googlegroups.com Subject: Patch 7.4.2189 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 7.4.2189 Problem: Cannot detect encoding in a fifo. Solution: Extend the stdin way of detecting encoding to fifo. Add a test for detecting encoding on stdin and fifo. (Ken Takata) Files: src/buffer.c, src/fileio.c, src/Makefile, src/testdir/Make_all.mak, src/testdir/test_startup_utf8.vim, src/vim.h *** ../vim-7.4.2188/src/buffer.c 2016-07-24 21:58:39.688057782 +0200 --- src/buffer.c 2016-08-09 22:08:40.448953913 +0200 *************** *** 70,75 **** --- 70,133 ---- /* Number of times free_buffer() was called. */ static int buf_free_count = 0; + /* Read data from buffer for retrying. */ + static int + read_buffer( + int read_stdin, /* read file from stdin, otherwise fifo */ + exarg_T *eap, /* for forced 'ff' and 'fenc' or NULL */ + int flags) /* extra flags for readfile() */ + { + int retval = OK; + linenr_T line_count; + + /* + * Read from the buffer which the text is already filled in and append at + * the end. This makes it possible to retry when 'fileformat' or + * 'fileencoding' was guessed wrong. + */ + line_count = curbuf->b_ml.ml_line_count; + retval = readfile( + read_stdin ? NULL : curbuf->b_ffname, + read_stdin ? NULL : curbuf->b_fname, + (linenr_T)line_count, (linenr_T)0, (linenr_T)MAXLNUM, eap, + flags | READ_BUFFER); + if (retval == OK) + { + /* Delete the binary lines. */ + while (--line_count >= 0) + ml_delete((linenr_T)1, FALSE); + } + else + { + /* Delete the converted lines. */ + while (curbuf->b_ml.ml_line_count > line_count) + ml_delete(line_count, FALSE); + } + /* Put the cursor on the first line. */ + curwin->w_cursor.lnum = 1; + curwin->w_cursor.col = 0; + + if (read_stdin) + { + /* Set or reset 'modified' before executing autocommands, so that + * it can be changed there. */ + if (!readonlymode && !bufempty()) + changed(); + else if (retval != FAIL) + unchanged(curbuf, FALSE); + + #ifdef FEAT_AUTOCMD + # ifdef FEAT_EVAL + apply_autocmds_retval(EVENT_STDINREADPOST, NULL, NULL, FALSE, + curbuf, &retval); + # else + apply_autocmds(EVENT_STDINREADPOST, NULL, NULL, FALSE, curbuf); + # endif + #endif + } + return retval; + } + /* * Open current buffer, that is: open the memfile and read the file into * memory. *************** *** 88,93 **** --- 146,152 ---- #ifdef FEAT_SYN_HL long old_tw = curbuf->b_p_tw; #endif + int read_fifo = FALSE; /* * The 'readonly' flag is only set when BF_NEVERLOADED is being reset. *************** *** 143,159 **** ) { int old_msg_silent = msg_silent; ! #ifdef FEAT_NETBEANS_INTG int oldFire = netbeansFireChanges; netbeansFireChanges = 0; #endif if (shortmess(SHM_FILEINFO)) msg_silent = 1; retval = readfile(curbuf->b_ffname, curbuf->b_fname, (linenr_T)0, (linenr_T)0, (linenr_T)MAXLNUM, eap, ! flags | READ_NEW); msg_silent = old_msg_silent; #ifdef FEAT_NETBEANS_INTG netbeansFireChanges = oldFire; --- 202,243 ---- ) { int old_msg_silent = msg_silent; ! #ifdef UNIX ! int save_bin = curbuf->b_p_bin; ! int perm; ! #endif #ifdef FEAT_NETBEANS_INTG int oldFire = netbeansFireChanges; netbeansFireChanges = 0; #endif + #ifdef UNIX + perm = mch_getperm(curbuf->b_ffname); + if (perm >= 0 && (0 + # ifdef S_ISFIFO + || S_ISFIFO(perm) + # endif + # ifdef S_ISSOCK + || S_ISSOCK(perm) + # endif + )) + read_fifo = TRUE; + if (read_fifo) + curbuf->b_p_bin = TRUE; + #endif if (shortmess(SHM_FILEINFO)) msg_silent = 1; retval = readfile(curbuf->b_ffname, curbuf->b_fname, (linenr_T)0, (linenr_T)0, (linenr_T)MAXLNUM, eap, ! flags | READ_NEW | (read_fifo ? READ_FIFO : 0)); ! #ifdef UNIX ! if (read_fifo) ! { ! curbuf->b_p_bin = save_bin; ! if (retval == OK) ! retval = read_buffer(FALSE, eap, flags); ! } ! #endif msg_silent = old_msg_silent; #ifdef FEAT_NETBEANS_INTG netbeansFireChanges = oldFire; *************** *** 164,171 **** } else if (read_stdin) { ! int save_bin = curbuf->b_p_bin; ! linenr_T line_count; /* * First read the text in binary mode into the buffer. --- 248,254 ---- } else if (read_stdin) { ! int save_bin = curbuf->b_p_bin; /* * First read the text in binary mode into the buffer. *************** *** 179,220 **** flags | (READ_NEW + READ_STDIN)); curbuf->b_p_bin = save_bin; if (retval == OK) ! { ! line_count = curbuf->b_ml.ml_line_count; ! retval = readfile(NULL, NULL, (linenr_T)line_count, ! (linenr_T)0, (linenr_T)MAXLNUM, eap, ! flags | READ_BUFFER); ! if (retval == OK) ! { ! /* Delete the binary lines. */ ! while (--line_count >= 0) ! ml_delete((linenr_T)1, FALSE); ! } ! else ! { ! /* Delete the converted lines. */ ! while (curbuf->b_ml.ml_line_count > line_count) ! ml_delete(line_count, FALSE); ! } ! /* Put the cursor on the first line. */ ! curwin->w_cursor.lnum = 1; ! curwin->w_cursor.col = 0; ! ! /* Set or reset 'modified' before executing autocommands, so that ! * it can be changed there. */ ! if (!readonlymode && !bufempty()) ! changed(); ! else if (retval != FAIL) ! unchanged(curbuf, FALSE); ! #ifdef FEAT_AUTOCMD ! # ifdef FEAT_EVAL ! apply_autocmds_retval(EVENT_STDINREADPOST, NULL, NULL, FALSE, ! curbuf, &retval); ! # else ! apply_autocmds(EVENT_STDINREADPOST, NULL, NULL, FALSE, curbuf); ! # endif ! #endif ! } } /* if first time loading this buffer, init b_chartab[] */ --- 262,268 ---- flags | (READ_NEW + READ_STDIN)); curbuf->b_p_bin = save_bin; if (retval == OK) ! retval = read_buffer(TRUE, eap, flags); } /* if first time loading this buffer, init b_chartab[] */ *************** *** 243,249 **** #endif ) changed(); ! else if (retval != FAIL && !read_stdin) unchanged(curbuf, FALSE); save_file_ff(curbuf); /* keep this fileformat */ --- 291,297 ---- #endif ) changed(); ! else if (retval != FAIL && !read_stdin && !read_fifo) unchanged(curbuf, FALSE); save_file_ff(curbuf); /* keep this fileformat */ *** ../vim-7.4.2188/src/fileio.c 2016-07-29 20:50:19.859640367 +0200 --- src/fileio.c 2016-08-09 22:08:40.448953913 +0200 *************** *** 212,217 **** --- 212,218 ---- * stdin) * READ_DUMMY read into a dummy buffer (to check if file contents changed) * READ_KEEP_UNDO don't clear undo info or read it from a file + * READ_FIFO read from fifo/socket instead of a file * * return FAIL for failure, OK otherwise */ *************** *** 231,236 **** --- 232,238 ---- int filtering = (flags & READ_FILTER); int read_stdin = (flags & READ_STDIN); int read_buffer = (flags & READ_BUFFER); + int read_fifo = (flags & READ_FIFO); int set_options = newfile || read_buffer || (eap != NULL && eap->read_edit); linenr_T read_buf_lnum = 1; /* next line to read from curbuf */ *************** *** 431,437 **** } } ! if (!read_stdin && !read_buffer) { #ifdef UNIX /* --- 433,439 ---- } } ! if (!read_stdin && !read_buffer && !read_fifo) { #ifdef UNIX /* *************** *** 489,495 **** if (check_readonly && !readonlymode) curbuf->b_p_ro = FALSE; ! if (newfile && !read_stdin && !read_buffer) { /* Remember time of file. */ if (mch_stat((char *)fname, &st) >= 0) --- 491,497 ---- if (check_readonly && !readonlymode) curbuf->b_p_ro = FALSE; ! if (newfile && !read_stdin && !read_buffer && !read_fifo) { /* Remember time of file. */ if (mch_stat((char *)fname, &st) >= 0) *************** *** 1101,1106 **** --- 1103,1109 ---- * and we can't do it internally or with iconv(). */ if (fio_flags == 0 && !read_stdin && !read_buffer && *p_ccv != NUL + && !read_fifo # ifdef USE_ICONV && iconv_fd == (iconv_t)-1 # endif *************** *** 1149,1155 **** /* Set "can_retry" when it's possible to rewind the file and try with * another "fenc" value. It's FALSE when no other "fenc" to try, reading * stdin or fixed at a specific encoding. */ ! can_retry = (*fenc != NUL && !read_stdin && !keep_dest_enc); #endif if (!skip_read) --- 1152,1158 ---- /* Set "can_retry" when it's possible to rewind the file and try with * another "fenc" value. It's FALSE when no other "fenc" to try, reading * stdin or fixed at a specific encoding. */ ! can_retry = (*fenc != NUL && !read_stdin && !read_fifo && !keep_dest_enc); #endif if (!skip_read) *************** *** 1166,1171 **** --- 1169,1175 ---- && curbuf->b_ffname != NULL && curbuf->b_p_udf && !filtering + && !read_fifo && !read_stdin && !read_buffer); if (read_undo_file) *************** *** 2666,2672 **** #endif #ifdef FEAT_AUTOCMD ! if (!read_stdin && !read_buffer) { int m = msg_scroll; int n = msg_scrolled; --- 2670,2676 ---- #endif #ifdef FEAT_AUTOCMD ! if (!read_stdin && !read_fifo && (!read_buffer || sfname != NULL)) { int m = msg_scroll; int n = msg_scrolled; *************** *** 2685,2691 **** if (filtering) apply_autocmds_exarg(EVENT_FILTERREADPOST, NULL, sfname, FALSE, curbuf, eap); ! else if (newfile) { apply_autocmds_exarg(EVENT_BUFREADPOST, NULL, sfname, FALSE, curbuf, eap); --- 2689,2695 ---- if (filtering) apply_autocmds_exarg(EVENT_FILTERREADPOST, NULL, sfname, FALSE, curbuf, eap); ! else if (newfile || (read_buffer && sfname != NULL)) { apply_autocmds_exarg(EVENT_BUFREADPOST, NULL, sfname, FALSE, curbuf, eap); *** ../vim-7.4.2188/src/Makefile 2016-08-08 22:26:37.989926409 +0200 --- src/Makefile 2016-08-09 22:02:09.444620033 +0200 *************** *** 2114,2119 **** --- 2114,2120 ---- test_signs \ test_sort \ test_startup \ + test_startup_utf8 \ test_stat \ test_statusline \ test_syn_attr \ *** ../vim-7.4.2188/src/testdir/Make_all.mak 2016-08-08 22:26:37.989926409 +0200 --- src/testdir/Make_all.mak 2016-08-09 22:02:09.444620033 +0200 *************** *** 185,190 **** --- 185,191 ---- test_ruby.res \ test_signs.res \ test_startup.res \ + test_startup_utf8.res \ test_stat.res \ test_syntax.res \ test_textobjects.res \ *** ../vim-7.4.2188/src/testdir/test_startup_utf8.vim 2016-08-09 22:12:38.374729774 +0200 --- src/testdir/test_startup_utf8.vim 2016-08-09 22:02:09.444620033 +0200 *************** *** 0 **** --- 1,64 ---- + " Tests for startup using utf-8. + if !has('multi_byte') + finish + endif + + source shared.vim + + func Test_read_stdin_utf8() + let linesin = ['テスト', '€ÀÈÌÒÙ'] + call writefile(linesin, 'Xtestin') + let before = [ + \ 'set enc=utf-8', + \ 'set fencs=cp932,utf-8', + \ ] + let after = [ + \ 'write ++enc=utf-8 Xtestout', + \ 'quit!', + \ ] + if has('win32') + let pipecmd = 'type Xtestin | ' + else + let pipecmd = 'cat Xtestin | ' + endif + if RunVimPiped(before, after, '-', pipecmd) + let lines = readfile('Xtestout') + call assert_equal(linesin, lines) + else + call assert_equal('', 'RunVimPiped failed.') + endif + call delete('Xtestout') + call delete('Xtestin') + endfunc + + func Test_read_fifo_utf8() + if !has('unix') + return + endif + " Using bash/zsh's process substitution. + if executable('bash') + set shell=bash + elseif executable('zsh') + set shell=zsh + else + return + endif + let linesin = ['テスト', '€ÀÈÌÒÙ'] + call writefile(linesin, 'Xtestin') + let before = [ + \ 'set enc=utf-8', + \ 'set fencs=cp932,utf-8', + \ ] + let after = [ + \ 'write ++enc=utf-8 Xtestout', + \ 'quit!', + \ ] + if RunVim(before, after, '<(cat Xtestin)') + let lines = readfile('Xtestout') + call assert_equal(linesin, lines) + else + call assert_equal('', 'RunVim failed.') + endif + call delete('Xtestout') + call delete('Xtestin') + endfunc *** ../vim-7.4.2188/src/vim.h 2016-08-07 15:19:22.204295369 +0200 --- src/vim.h 2016-08-09 22:02:09.448619995 +0200 *************** *** 980,986 **** #define READ_STDIN 0x04 /* read from stdin */ #define READ_BUFFER 0x08 /* read from curbuf (converting stdin) */ #define READ_DUMMY 0x10 /* reading into a dummy buffer */ ! #define READ_KEEP_UNDO 0x20 /* keep undo info*/ /* Values for change_indent() */ #define INDENT_SET 1 /* set indent */ --- 980,987 ---- #define READ_STDIN 0x04 /* read from stdin */ #define READ_BUFFER 0x08 /* read from curbuf (converting stdin) */ #define READ_DUMMY 0x10 /* reading into a dummy buffer */ ! #define READ_KEEP_UNDO 0x20 /* keep undo info */ ! #define READ_FIFO 0x40 /* read from fifo or socket */ /* Values for change_indent() */ #define INDENT_SET 1 /* set indent */ *** ../vim-7.4.2188/src/version.c 2016-08-09 21:51:36.158407578 +0200 --- src/version.c 2016-08-09 22:09:01.500756922 +0200 *************** *** 765,766 **** --- 765,768 ---- { /* Add new patch number below this line */ + /**/ + 2189, /**/ -- Q: What's a light-year? A: One-third less calories than a regular year. /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///