Cache read values in local state, handle mutable variables properly

REFERENCE.md

@@ -24,9 +24,9 @@ Another line.
 
 #### Checkpoints
 
-When executing a piece of Anselme code, it will not directly modify the global state (i.e. the values of variables used by every script), but only locally, in this execution.
+When executing a piece of Anselme code, your scripts will not directly modify the global state (i.e. the values of variables used by every script), but only locally, in its associated interpreter instance. Meaning that if you change a variable, its value will only be set in the local state, so the new value will be accessible from the current interpreter but not other interpreters, at least until the next checkpoint. Similarly, the first time your script reads a variable its value at this time is kept into the local state so it can not be affected by other scripts, at least until the next checkpoint.
 
-Right after reaching a checkpoint line, Anselme will merge the local state with the global one, i.e., make every change accessible to other scripts.
+Right after reaching a checkpoint line, Anselme will merge the local state with the global one, i.e., make every change accessible to other scripts, and get checkpointed changes from other scripts.
 
 ```
 $ main
@@ -51,7 +51,9 @@ $ parallel
 ~ main
 ```
 
-The purpose of this system is both to allow several scripts to run at the same time with an easy way to avoid interferences, and to make sure the global state is always in a consistent (and not in the middle of a calculation): since scripts can be interrupted at any time, when it is interrupted, anything that was changed between the last checkpoint and the interruption will be discarded. When running the script again, it will resume correctly at the last reached checkpoint. See [function calls](#function-calls) for more details on how to call/resume a function.
+The purpose of this system is both to allow several scripts to run at the same time with an easy way to avoid interferences, and to make sure the global state is always in a consistent (and not in the middle of a calculation): since scripts can be interrupted at any time, when it is interrupted, anything that was changed between the last checkpoint and the interruption will be discarded. If you're a RDBMS person, that's more-or-less equivalent to a transaction with a repeatable read isolation level (without any sort of locking or lost update protection though).
+
+When running the script again, it will resume correctly at the last reached checkpoint. See [function calls](#function-calls) for more details on how to call/resume a function.
 
 Checkpoints are set per function, and are expected to be defined inside functions only.
 

anselme.lua

@@ -6,11 +6,11 @@ local anselme = {
 	-- api is incremented a each update which may break Lua API compatibility
 	versions = {
 		save = 1,
-		language = 18,
-		api = 2
+		language = 19,
+		api = 3
 	},
 	-- version is incremented at each update
-	version = 19,
+	version = 20,
 	--- currently running interpreter
 	running = nil
 }
@@ -60,6 +60,25 @@ local function is_file(path)
 	end
 end
 
+--- recursively copy a table, handle cyclic references, no metatable
+local function copy(t, cache)
+	if type(t) == "table" then
+		cache = cache or {}
+		if cache[t] then
+			return cache[t]
+		else
+			local c = {}
+			cache[t] = c
+			for k, v in pairs(t) do
+				c[k] = copy(v, cache)
+			end
+			return c
+		end
+	else
+		return t
+	end
+end
+
 --- interpreter methods
 local interpreter_methods = {
 	-- interpreter state
@@ -474,7 +493,16 @@ local vm_mt = {
 				builtin_aliases = self.state.builtin_aliases,
 				aliases = setmetatable({}, { __index = self.state.aliases }),
 				functions = self.state.functions, -- no need for a cache as we can't define or modify any function from the interpreter for now
-				variables = setmetatable({}, { __index = self.state.variables }),
+				variables = setmetatable({}, {
+					__index = function(variables, k)
+						local cache = getmetatable(variables).cache
+						if cache[k] == nil then
+							cache[k] = copy(self.state.variables[k])
+						end
+						return cache[k]
+					end,
+					cache = {} -- cache of previously read values, to get repeatable reads & handle mutable types without changing global state
+				}),
 				interpreter = {
 					-- constant
 					global_state = self.state,

interpreter/common.lua

@@ -32,11 +32,23 @@ local common
 common = {
 	--- merge interpreter state with global state
 	merge_state = function(state)
+		-- merge alias state
 		local global = state.interpreter.global_state
 		for alias, fqm in pairs(state.aliases) do
 			global.aliases[alias] = fqm
 			state.aliases[alias] = nil
 		end
+		-- variable state
+		-- move values modifed in-place from read cache to variables
+		local cache = getmetatable(state.variables).cache
+		for var, value in pairs(cache) do
+			if value.modified then
+				value.modified = nil
+				state.variables[var] = value
+			end
+			cache[var] = nil
+		end
+		-- merge modified variables
 		for var, value in pairs(state.variables) do
 			global.variables[var] = value
 			state.variables[var] = nil

stdlib/functions.lua

@@ -91,6 +91,7 @@ functions = {
 	["()(l::list, i::number) := v"] = {
 		mode = "raw",
 		value = function(l, i, v)
+			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			local iv = i.type == "type" and i.value[1] or i
 			lv.value[iv.value] = v
@@ -100,6 +101,7 @@ functions = {
 	["()(l::list, k::string) := v"] = {
 		mode = "raw",
 		value = function(l, k, v)
+			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			local kv = k.type == "type" and k.value[1] or k
 			-- update index
@@ -168,27 +170,33 @@ functions = {
 	["insert(l::list, v)"] = {
 		mode = "raw",
 		value = function(l, v)
+			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			table.insert(lv.value, v)
+			return l
 		end
 	},
 	["insert(l::list, i::number, v)"] = {
 		mode = "raw",
 		value = function(l, i, v)
+			l.modified = true
 			local lv = l.type == "type" and l.value[1] or l
 			local iv = i.type == "type" and i.value[1] or i
 			table.insert(lv.value, iv.value, v)
+			return l
 		end
 	},
 	["remove(l::list)"] = {
 		mode = "untyped raw",
 		value = function(l)
+			l.modified = true
 			return table.remove(l.value)
 		end
 	},
 	["remove(l::list, i::number)"] = {
 		mode = "untyped raw",
 		value = function(l, i)
+			l.modified = true
 			return table.remove(l.value, i.value)
 		end
 	},

test/run.lua

@@ -195,6 +195,19 @@ else
 					local t, d = istate:step()
 					table.insert(result, { t, d })
 				until t == "return" or t == "error"
+
+				local postrun = vm:eval(namespace..".post run")
+				if postrun then
+					istate, e = vm:run(namespace.."."..postrun)
+					if not istate then
+						table.insert(result, { "error", e })
+					else
+						repeat
+							local t, d = istate:step()
+							table.insert(result, { t, d })
+						until t == "return" or t == "error"
+					end
+				end
 			end
 		else
 			table.insert(result, { "error", err })

test/tests/checkpoint merging mutable value.ans

@@ -0,0 +1,26 @@
+:post run = "after error"
+
+:l = [1,2]
+
+1,2: {l}
+
+~ l.insert(3)
+
+1,2,3: {l}
+
+§ a
+
+~ l.insert(4)
+
+1,2,3,4: {l}
+
+§ b
+
+~ l.insert(5)
+
+1,2,3,4,5: {l}
+
+~ error("cancel merge")
+
+$ after error
+	1,2,3,4: {l}

test/tests/checkpoint merging mutable value.lua

@@ -0,0 +1,68 @@
+local _={}
+_[27]={}
+_[26]={}
+_[25]={}
+_[24]={}
+_[23]={}
+_[22]={tags=_[27],text="[1, 2, 3, 4]"}
+_[21]={tags=_[27],text="1,2,3,4: "}
+_[20]={tags=_[26],text="[1, 2, 3, 4, 5]"}
+_[19]={tags=_[26],text="1,2,3,4,5: "}
+_[18]={tags=_[25],text="[1, 2, 3, 4]"}
+_[17]={tags=_[25],text="1,2,3,4: "}
+_[16]={tags=_[24],text="[1, 2, 3]"}
+_[15]={tags=_[24],text="1,2,3: "}
+_[14]={tags=_[23],text="[1, 2]"}
+_[13]={tags=_[23],text="1,2: "}
+_[12]={_[21],_[22]}
+_[11]={_[19],_[20]}
+_[10]={_[17],_[18]}
+_[9]={_[15],_[16]}
+_[8]={_[13],_[14]}
+_[7]={"return"}
+_[6]={"text",_[12]}
+_[5]={"error","cancel merge; in Lua function \"error\"; at test/tests/checkpoint merging mutable value.ans:23"}
+_[4]={"text",_[11]}
+_[3]={"text",_[10]}
+_[2]={"text",_[9]}
+_[1]={"text",_[8]}
+return {_[1],_[2],_[3],_[4],_[5],_[6],_[7]}
+--[[
+{ "text", { {
+      tags = <1>{},
+      text = "1,2: "
+    }, {
+      tags = <table 1>,
+      text = "[1, 2]"
+    } } }
+{ "text", { {
+      tags = <1>{},
+      text = "1,2,3: "
+    }, {
+      tags = <table 1>,
+      text = "[1, 2, 3]"
+    } } }
+{ "text", { {
+      tags = <1>{},
+      text = "1,2,3,4: "
+    }, {
+      tags = <table 1>,
+      text = "[1, 2, 3, 4]"
+    } } }
+{ "text", { {
+      tags = <1>{},
+      text = "1,2,3,4,5: "
+    }, {
+      tags = <table 1>,
+      text = "[1, 2, 3, 4, 5]"
+    } } }
+{ "error", 'cancel merge; in Lua function "error"; at test/tests/checkpoint merging mutable value.ans:23' }
+{ "text", { {
+      tags = <1>{},
+      text = "1,2,3,4: "
+    }, {
+      tags = <table 1>,
+      text = "[1, 2, 3, 4]"
+    } } }
+{ "return" }
+]]--

test/tests/checkpoint merging variable.ans

@@ -0,0 +1,26 @@
+:post run = "after error"
+
+:l = 1
+
+1: {l}
+
+~ l := 2
+
+2: {l}
+
+§ a
+
+~ l := 3
+
+3: {l}
+
+§ b
+
+~ l := 4
+
+4: {l}
+
+~ error("cancel merge")
+
+$ after error
+	3: {l}

test/tests/checkpoint merging variable.lua

@@ -0,0 +1,68 @@
+local _={}
+_[27]={}
+_[26]={}
+_[25]={}
+_[24]={}
+_[23]={}
+_[22]={tags=_[27],text="3"}
+_[21]={tags=_[27],text="3: "}
+_[20]={tags=_[26],text="4"}
+_[19]={tags=_[26],text="4: "}
+_[18]={tags=_[25],text="3"}
+_[17]={tags=_[25],text="3: "}
+_[16]={tags=_[24],text="2"}
+_[15]={tags=_[24],text="2: "}
+_[14]={tags=_[23],text="1"}
+_[13]={tags=_[23],text="1: "}
+_[12]={_[21],_[22]}
+_[11]={_[19],_[20]}
+_[10]={_[17],_[18]}
+_[9]={_[15],_[16]}
+_[8]={_[13],_[14]}
+_[7]={"return"}
+_[6]={"text",_[12]}
+_[5]={"error","cancel merge; in Lua function \"error\"; at test/tests/checkpoint merging variable.ans:23"}
+_[4]={"text",_[11]}
+_[3]={"text",_[10]}
+_[2]={"text",_[9]}
+_[1]={"text",_[8]}
+return {_[1],_[2],_[3],_[4],_[5],_[6],_[7]}
+--[[
+{ "text", { {
+      tags = <1>{},
+      text = "1: "
+    }, {
+      tags = <table 1>,
+      text = "1"
+    } } }
+{ "text", { {
+      tags = <1>{},
+      text = "2: "
+    }, {
+      tags = <table 1>,
+      text = "2"
+    } } }
+{ "text", { {
+      tags = <1>{},
+      text = "3: "
+    }, {
+      tags = <table 1>,
+      text = "3"
+    } } }
+{ "text", { {
+      tags = <1>{},
+      text = "4: "
+    }, {
+      tags = <table 1>,
+      text = "4"
+    } } }
+{ "error", 'cancel merge; in Lua function "error"; at test/tests/checkpoint merging variable.ans:23' }
+{ "text", { {
+      tags = <1>{},
+      text = "3: "
+    }, {
+      tags = <table 1>,
+      text = "3"
+    } } }
+{ "return" }
+]]--