feat(decode_string): decode binary string with NULs to Blob

seandewar · seandewar · commit de9df825d5e3 · 2021-09-15T21:19:30.000+01:00
Strings that previously decoded into a msgpack special for representing
BINs with NULs now convert to Blobs. It shouldn't be possible to decode
into this special anymore after this change?

Notably, Lua strings with NULs now convert to Blobs when passed to VimL.
diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
@@ -6750,9 +6750,9 @@ msgpackparse({list})					*msgpackparse()*
 			zero byte or if string is a mapping key and mapping is
 			being represented as special dictionary for other
 			reasons.
-		binary	|readfile()|-style list of strings. This value will
-			appear in |msgpackparse()| output if binary string
-			contains zero byte.
+		binary	|String|, or |Blob| if binary string contains zero
+			byte. This value cannot appear in |msgpackparse()|
+			output since blobs were introduced.
 		array	|List|. This value cannot appear in |msgpackparse()|
 			output.
 							*msgpack-special-map*
diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt
@@ -326,7 +326,8 @@ semantically equivalent in Lua to:
     end
 
 Lua nils, numbers, strings, tables and booleans are converted to their
-respective Vimscript types. Conversion of other Lua types is an error.
+respective Vimscript types. If a Lua string contains a NUL byte, it will be
+converted to a |Blob|. Conversion of other Lua types is an error.
 
 The magic global "_A" contains the second argument to luaeval().
 
diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c
@@ -246,14 +246,15 @@ list_T *decode_create_map_special_dict(typval_T *const ret_tv,
 /// Convert char* string to typval_T
 ///
 /// Depending on whether string has (no) NUL bytes, it may use a special
-/// dictionary or decode string to VAR_STRING.
+/// dictionary, VAR_BLOB, or decode string to VAR_STRING.
 ///
 /// @param[in]  s  String to decode.
 /// @param[in]  len  String length.
 /// @param[in]  hasnul  Whether string has NUL byte, not or it was not yet
 ///                     determined.
-/// @param[in]  binary  If true, save special string type as kMPBinary,
-///                     otherwise kMPString.
+/// @param[in]  binary  Determines decode type if string has NUL bytes.
+///                     If true convert string to VAR_BLOB, otherwise to the
+///                     kMPString special type.
 /// @param[in]  s_allocated  If true, then `s` was allocated and can be saved in
 ///                          a returned structure. If it is not saved there, it
 ///                          will be freed.
@@ -269,21 +270,28 @@ typval_T decode_string(const char *const s, const size_t len,
                               ? ((s != NULL) && (memchr(s, NUL, len) != NULL))
                               : (bool)hasnul);
   if (really_hasnul) {
-    list_T *const list = tv_list_alloc(kListLenMayKnow);
-    tv_list_ref(list);
     typval_T tv;
-    create_special_dict(&tv, binary ? kMPBinary : kMPString, ((typval_T) {
-      .v_type = VAR_LIST,
-      .v_lock = VAR_UNLOCKED,
-      .vval = { .v_list = list },
-    }));
-    const int elw_ret = encode_list_write((void *)list, s, len);
-    if (s_allocated) {
-      xfree((void *)s);
-    }
-    if (elw_ret == -1) {
-      tv_clear(&tv);
-      return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED };
+    tv.v_lock = VAR_UNLOCKED;
+    if (binary) {
+      tv_blob_alloc_ret(&tv);
+      ga_concat_len(&tv.vval.v_blob->bv_ga, s, len);
+    } else {
+      list_T *const list = tv_list_alloc(kListLenMayKnow);
+      tv_list_ref(list);
+      create_special_dict(&tv, kMPString,
+                          ((typval_T){
+                              .v_type = VAR_LIST,
+                              .v_lock = VAR_UNLOCKED,
+                              .vval = { .v_list = list },
+                          }));
+      const int elw_ret = encode_list_write((void *)list, s, len);
+      if (s_allocated) {
+        xfree((void *)s);
+      }
+      if (elw_ret == -1) {
+        tv_clear(&tv);
+        return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED };
+      }
     }
     return tv;
   } else {
diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua
@@ -364,8 +364,7 @@ describe('msgpack*() functions', function()
     command('let dumped = ["\\xC4\\x01\\n"]')
     command('let parsed = msgpackparse(dumped)')
     command('let dumped2 = msgpackdump(parsed)')
-    eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed'))
-    eq(1, eval('parsed[0]._TYPE is v:msgpack_types.binary'))
+    eq({'\000'}, eval('parsed'))
     eq(1, eval('dumped ==# dumped2'))
   end)
 
diff --git a/test/functional/lua/api_spec.lua b/test/functional/lua/api_spec.lua
@@ -15,15 +15,15 @@ describe('luaeval(vim.api.…)', function()
     describe('nvim_buf_get_lines', function()
       it('works', function()
         funcs.setline(1, {"abc", "def", "a\nb", "ttt"})
-        eq({{_TYPE={}, _VAL={'a\nb'}}},
+        eq({'a\000b'},
            funcs.luaeval('vim.api.nvim_buf_get_lines(1, 2, 3, false)'))
       end)
     end)
     describe('nvim_buf_set_lines', function()
       it('works', function()
         funcs.setline(1, {"abc", "def", "a\nb", "ttt"})
         eq(NIL, funcs.luaeval('vim.api.nvim_buf_set_lines(1, 1, 2, false, {"b\\0a"})'))
-        eq({'abc', {_TYPE={}, _VAL={'b\na'}}, {_TYPE={}, _VAL={'a\nb'}}, 'ttt'},
+        eq({'abc', 'b\000a', 'a\000b', 'ttt'},
            funcs.luaeval('vim.api.nvim_buf_get_lines(1, 0, 4, false)'))
       end)
     end)
@@ -68,6 +68,7 @@ describe('luaeval(vim.api.…)', function()
     eq({}, funcs.luaeval('vim.api.nvim_eval("[]")'))
     eq({}, funcs.luaeval('vim.api.nvim_eval("{}")'))
     eq(1, funcs.luaeval('vim.api.nvim_eval("1.0")'))
+    eq('\000', funcs.luaeval('vim.api.nvim_eval("0z00")'))
     eq(true, funcs.luaeval('vim.api.nvim_eval("v:true")'))
     eq(false, funcs.luaeval('vim.api.nvim_eval("v:false")'))
     eq(NIL, funcs.luaeval('vim.api.nvim_eval("v:null")'))
diff --git a/test/functional/lua/luaeval_spec.lua b/test/functional/lua/luaeval_spec.lua
@@ -63,11 +63,10 @@ describe('luaeval()', function()
       eq('\n[[...@0]]', funcs.execute('echo luaeval("l")'))
     end)
   end)
-  describe('strings', function()
-    it('are successfully converted to special dictionaries', function()
+  describe('strings with NULs', function()
+    it('are successfully converted to blobs', function()
       command([[let s = luaeval('"\0"')]])
-      eq({_TYPE={}, _VAL={'\n'}}, meths.get_var('s'))
-      eq(1, funcs.eval('s._TYPE is v:msgpack_types.binary'))
+      eq('\000', meths.get_var('s'))
     end)
     it('are successfully converted to special dictionaries in table keys',
     function()
@@ -76,13 +75,10 @@ describe('luaeval()', function()
       eq(1, funcs.eval('d._TYPE is v:msgpack_types.map'))
       eq(1, funcs.eval('d._VAL[0][0]._TYPE is v:msgpack_types.string'))
     end)
-    it('are successfully converted to special dictionaries from a list',
+    it('are successfully converted to blobs from a list',
     function()
       command([[let l = luaeval('{"abc", "a\0b", "c\0d", "def"}')]])
-      eq({'abc', {_TYPE={}, _VAL={'a\nb'}}, {_TYPE={}, _VAL={'c\nd'}}, 'def'},
-         meths.get_var('l'))
-      eq(1, funcs.eval('l[1]._TYPE is v:msgpack_types.binary'))
-      eq(1, funcs.eval('l[2]._TYPE is v:msgpack_types.binary'))
+      eq({'abc', 'a\000b', 'c\000d', 'def'}, meths.get_var('l'))
     end)
   end)
 
@@ -100,9 +96,9 @@ describe('luaeval()', function()
     eq(1, eval('type(luaeval("\'test\'"))'))
 
     eq('', funcs.luaeval('""'))
-    eq({_TYPE={}, _VAL={'\n'}}, funcs.luaeval([['\0']]))
-    eq({_TYPE={}, _VAL={'\n', '\n'}}, funcs.luaeval([['\0\n\0']]))
-    eq(1, eval([[luaeval('"\0\n\0"')._TYPE is v:msgpack_types.binary]]))
+    eq('\000', funcs.luaeval([['\0']]))
+    eq('\000\n\000', funcs.luaeval([['\0\n\0']]))
+    eq(10, eval([[type(luaeval("'\\0\\n\\0'"))]]))
 
     eq(true, funcs.luaeval('true'))
     eq(false, funcs.luaeval('false'))
@@ -122,12 +118,11 @@ describe('luaeval()', function()
     local level = 30
     eq(nested_by_level[level].o, funcs.luaeval(nested_by_level[level].s))
 
-    eq({_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, {_TYPE={}, _VAL={'\n', '\n\n'}}}}},
+    eq({_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, '\000\n\000\000'}}},
        funcs.luaeval([[{['\0\n\0']='\0\n\0\0'}]]))
     eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._TYPE is v:msgpack_types.map]]))
     eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._VAL[0][0]._TYPE is v:msgpack_types.string]]))
-    eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._VAL[0][1]._TYPE is v:msgpack_types.binary]]))
-    eq({nested={{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, {_TYPE={}, _VAL={'\n', '\n\n'}}}}}}},
+    eq({nested={{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, '\000\n\000\000'}}}}},
        funcs.luaeval([[{nested={{['\0\n\0']='\0\n\0\0'}}}]]))
   end)
 
@@ -175,8 +170,8 @@ describe('luaeval()', function()
   end
 
   it('correctly passes special dictionaries', function()
-    eq({'binary', {'\n', '\n'}}, luaevalarg(sp('binary', '["\\n", "\\n"]')))
-    eq({'binary', {'\n', '\n'}}, luaevalarg(sp('string', '["\\n", "\\n"]')))
+    eq({0, '\000\n\000'}, luaevalarg(sp('binary', '["\\n", "\\n"]')))
+    eq({0, '\000\n\000'}, luaevalarg(sp('string', '["\\n", "\\n"]')))
     eq({0, true}, luaevalarg(sp('boolean', 1)))
     eq({0, false}, luaevalarg(sp('boolean', 0)))
     eq({0, NIL}, luaevalarg(sp('nil', 0)))
