Proper checkpointing of mutable values

REFERENCE.md

@@ -49,6 +49,8 @@ $ parallel
     parallel: {main.var}
 
 ~ main
+
+(note: if two scripts try to modify the same value at the same time, one of them will win, but which one is undefined/a surprise)
 ```
 
 The purpose of this system is both to allow several scripts to run at the same time with an easy way to avoid interferences, and to make sure the global state is always in a consistent (and not in the middle of a calculation): since scripts can be interrupted at any time, when it is interrupted, anything that was changed between the last checkpoint and the interruption will be discarded. If you're a RDBMS person, that's more-or-less equivalent to a transaction with a repeatable read isolation level (without any sort of locking or lost update protection though).

anselme.lua

@@ -479,11 +479,13 @@ local vm_mt = {
 					__index = function(variables, k)
 						local cache = getmetatable(variables).cache
 						if cache[k] == nil then
-							cache[k] = copy(self.state.variables[k])
+							cache[k] = copy(self.state.variables[k], getmetatable(variables).copy_cache)
 						end
 						return cache[k]
 					end,
-					cache = {} -- cache of previously read values, to get repeatable reads & handle mutable types without changing global state
+					copy_cache = {}, -- table of [original table] = copied table
+					modified_tables = {}, -- list of modified tables (copies) that should be merged with global state on next checkpoint
+					cache = {} -- cache of previously read values (copies), to get repeatable reads & handle mutable types without changing global state
 				}),
 				interpreter = {
 					-- constant

common.lua

@@ -1,6 +1,20 @@
+--- replace values recursively in table t according to to_replace ([old table] = new table)
+local function replace_in_table(t, to_replace, already_replaced)
+	already_replaced = already_replaced or {}
+	already_replaced[t] = true
+	for k, v in pairs(t) do
+		if to_replace[v] then
+			t[k] = to_replace[v]
+		elseif type(v) == "table" and not already_replaced[v] then
+			replace_in_table(v, to_replace, already_replaced)
+		end
+	end
+end
+
 local common
 common = {
-	--- recursively copy a table, handle cyclic references, no metatable
+	--- recursively copy a table, handle cyclic references, no metatable, don't copy keys
+	-- cache is table with copied tables [original table] = copied value, will use temporary table is omitted
 	copy = function(t, cache)
 		if type(t) == "table" then
 			cache = cache or {}
@@ -17,6 +31,33 @@ common = {
 		else
 			return t
 		end
+	end,
+	--- given a table t from which some copy was issued, the copy cache, and a list of tables from the copied version,
+	-- put theses copied tables in t in place of their original values, preserving references to non-modified values
+	replace_with_copied_values = function(t, cache, copied_to_replace)
+		-- reverse copy cache
+		local ehcac = {}
+		for k, v in pairs(cache) do ehcac[v] = k end
+		-- build table of [original table] = replacement copied table
+		local to_replace = {}
+		for _, v in ipairs(copied_to_replace) do
+			local original = ehcac[v]
+			if original then -- table doesn't have an original value if it's a new table...
+				to_replace[original] = v
+			end
+		end
+		-- fix references to not-modified tables in modified values
+		local not_modified = {}
+		for original, modified in pairs(cache) do
+			if not to_replace[original] then
+				not_modified[modified] = original
+			end
+		end
+		for _, m in ipairs(copied_to_replace) do
+			replace_in_table(m, not_modified)
+		end
+		-- replace
+		replace_in_table(t, to_replace)
 	end
 }
 return common

interpreter/common.lua

@@ -1,6 +1,6 @@
 local atypes, ltypes
 local eval, run_block
-local copy
+local replace_with_copied_values
 local common
 
 --- copy some text & process it to be suited to be sent to Lua in an event
@@ -49,17 +49,13 @@ common = {
 			global.aliases[alias] = fqm
 			state.aliases[alias] = nil
 		end
-		-- variable state
-		-- move values modifed in-place from read cache to variables
-		local cache = getmetatable(state.variables).cache
-		for var, value in pairs(cache) do
-			if value.modified then
-				value.modified = nil
-				state.variables[var] = value
-			end
-			cache[var] = nil
-		end
-		-- merge modified variables
+		-- merge modified mutable varables
+		local mt = getmetatable(state.variables)
+		replace_with_copied_values(global.variables, mt.copy_cache, mt.modified_tables)
+		mt.copy_cache = {}
+		mt.modified = {}
+		mt.cache = {}
+		-- merge modified re-assigned variables
 		for var, value in pairs(state.variables) do
 			global.variables[var] = value
 			state.variables[var] = nil
@@ -109,6 +105,23 @@ common = {
 				end
 			end
 			return true
+		elseif a.type == "function reference" then
+			if #a.value ~= #b.value then
+				return false
+			end
+			for _, aname in ipairs(a.value) do
+				local found = false
+				for _, bname in ipairs(b.value) do
+					if aname == bname then
+						found = true
+						break
+					end
+				end
+				if not found then
+					return false
+				end
+			end
+			return true
 		else
 			return a.value == b.value
 		end
@@ -394,6 +407,6 @@ local types = require((...):gsub("interpreter%.common$", "stdlib.types"))
 atypes, ltypes = types.anselme, types.lua
 eval = require((...):gsub("common$", "expression"))
 run_block = require((...):gsub("common$", "interpreter")).run_block
-copy = require((...):gsub("interpreter%.common$", "common")).copy
+replace_with_copied_values = require((...):gsub("interpreter%.common$", "common")).replace_with_copied_values
 
 return common

stdlib/functions.lua

@@ -1,5 +1,10 @@
 local truthy, anselme, compare, is_of_type, identifier_pattern, format_identifier, find, get_variable
 
+local function mark_as_modified(v)
+	local modified = getmetatable(anselme.running.state.variables).modified_tables
+	table.insert(modified, v)
+end
+
 local functions
 functions = {
 	-- discard left
@@ -107,23 +112,23 @@ functions = {
 	["()(l::list, i::number) := v"] = {
 		mode = "raw",
 		value = function(l, i, v)
-			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			local iv = i.type == "type" and i.value[1] or i
 			lv.value[iv.value] = v
+			mark_as_modified(lv.value)
 			return v
 		end
 	},
 	["()(l::list, k::string) := v"] = {
 		mode = "raw",
 		value = function(l, k, v)
-			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			local kv = k.type == "type" and k.value[1] or k
 			-- update index
 			for _, x in ipairs(lv.value) do
 				if x.type == "pair" and compare(x.value[1], kv) then
 					x.value[2] = v
+					mark_as_modified(x.value)
 					return v
 				end
 			end
@@ -132,6 +137,7 @@ functions = {
 				type = "pair",
 				value = { kv, v }
 			})
+			mark_as_modified(lv.value)
 			return v
 		end
 	},
@@ -192,33 +198,33 @@ functions = {
 	["insert(l::list, v)"] = {
 		mode = "raw",
 		value = function(l, v)
-			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			table.insert(lv.value, v)
+			mark_as_modified(lv.value)
 			return l
 		end
 	},
 	["insert(l::list, i::number, v)"] = {
 		mode = "raw",
 		value = function(l, i, v)
-			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			local iv = i.type == "type" and i.value[1] or i
 			table.insert(lv.value, iv.value, v)
+			mark_as_modified(lv.value)
 			return l
 		end
 	},
 	["remove(l::list)"] = {
 		mode = "untyped raw",
 		value = function(l)
-			l.modified = true
+			mark_as_modified(l.value)
 			return table.remove(l.value)
 		end
 	},
 	["remove(l::list, i::number)"] = {
 		mode = "untyped raw",
 		value = function(l, i)
-			l.modified = true
+			mark_as_modified(l.value)
 			return table.remove(l.value, i.value)
 		end
 	},

test/run.lua

@@ -44,7 +44,7 @@ local function write_result(filebase, result)
 	o:write(ser(result))
 	o:write("\n--[[\n")
 	for _, v in ipairs(result) do
-		o:write(inspect(v).."\n")
+		o:write(inspect(v):gsub("]]", "] ]").."\n") -- professional-level bandaid when ]] appear in the output
 	end
 	o:write("]]--")
 	o:close()

test/tests/merge nested mutable bis.ans

@@ -0,0 +1,17 @@
+:post run = "check"
+
+:a = [1]
+:b = [2]
+
+~ a!insert(b)
+
+§ c
+
+~ b!insert(3)
+
+§ d
+
+~ b!insert(4)
+
+$ check
+	\[1,\[2,3,4]]: {a}

test/tests/merge nested mutable bis.lua

@@ -0,0 +1,21 @@
+local _={}
+_[8]={}
+_[7]={}
+_[6]={text="[1, [2, 3, 4]]",tags=_[8]}
+_[5]={text="[1,[2,3,4]]: ",tags=_[7]}
+_[4]={_[5],_[6]}
+_[3]={"return"}
+_[2]={"text",_[4]}
+_[1]={"return"}
+return {_[1],_[2],_[3]}
+--[[
+{ "return" }
+{ "text", { {
+      tags = {},
+      text = "[1,[2,3,4] ]: "
+    }, {
+      tags = {},
+      text = "[1, [2, 3, 4] ]"
+    } } }
+{ "return" }
+]]--

test/tests/merge nested mutable error bis.ans

@@ -0,0 +1,19 @@
+:post run = "check"
+
+:a = [1]
+:b = [2]
+
+~ a!insert(b)
+
+§ c
+
+~ b!insert(3)
+
+§ d
+
+~ b!insert(4)
+
+~ error("abort")
+
+$ check
+	\[1,\[2,3]]: {a}

test/tests/merge nested mutable error bis.lua

@@ -0,0 +1,21 @@
+local _={}
+_[8]={}
+_[7]={}
+_[6]={text="[1, [2, 3]]",tags=_[8]}
+_[5]={text="[1,[2,3]]: ",tags=_[7]}
+_[4]={_[5],_[6]}
+_[3]={"return"}
+_[2]={"text",_[4]}
+_[1]={"error","abort; in Lua function \"error\"; at test/tests/merge nested mutable error bis.ans:16"}
+return {_[1],_[2],_[3]}
+--[[
+{ "error", 'abort; in Lua function "error"; at test/tests/merge nested mutable error bis.ans:16' }
+{ "text", { {
+      tags = {},
+      text = "[1,[2,3] ]: "
+    }, {
+      tags = {},
+      text = "[1, [2, 3] ]"
+    } } }
+{ "return" }
+]]--

test/tests/merge nested mutable error.ans

@@ -0,0 +1,19 @@
+:post run = "check"
+
+:a = [1]
+:b = [2]
+
+~ a!insert(b)
+
+§ c
+
+~ b!insert(3)
+
+§ d
+
+~ a!insert(4)
+
+~ error("abort")
+
+$ check
+	\[1,\[2,3]]: {a}

test/tests/merge nested mutable error.lua

@@ -0,0 +1,21 @@
+local _={}
+_[8]={}
+_[7]={}
+_[6]={text="[1, [2, 3]]",tags=_[8]}
+_[5]={text="[1,[2,3]]: ",tags=_[7]}
+_[4]={_[5],_[6]}
+_[3]={"return"}
+_[2]={"text",_[4]}
+_[1]={"error","abort; in Lua function \"error\"; at test/tests/merge nested mutable error.ans:16"}
+return {_[1],_[2],_[3]}
+--[[
+{ "error", 'abort; in Lua function "error"; at test/tests/merge nested mutable error.ans:16' }
+{ "text", { {
+      tags = {},
+      text = "[1,[2,3] ]: "
+    }, {
+      tags = {},
+      text = "[1, [2, 3] ]"
+    } } }
+{ "return" }
+]]--

test/tests/merge nested mutable.ans

@@ -0,0 +1,17 @@
+:post run = "check"
+
+:a = [1]
+:b = [2]
+
+~ a!insert(b)
+
+§ c
+
+~ b!insert(3)
+
+§ d
+
+~ a!insert(4)
+
+$ check
+	\[1,\[2,3],4]: {a}

test/tests/merge nested mutable.lua

@@ -0,0 +1,21 @@
+local _={}
+_[8]={}
+_[7]={}
+_[6]={text="[1, [2, 3], 4]",tags=_[8]}
+_[5]={text="[1,[2,3],4]: ",tags=_[7]}
+_[4]={_[5],_[6]}
+_[3]={"return"}
+_[2]={"text",_[4]}
+_[1]={"return"}
+return {_[1],_[2],_[3]}
+--[[
+{ "return" }
+{ "text", { {
+      tags = {},
+      text = "[1,[2,3],4]: "
+    }, {
+      tags = {},
+      text = "[1, [2, 3], 4]"
+    } } }
+{ "return" }
+]]--