From 581c60048d4052b36ae27014abcb8195109f1688 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89tienne=20Reuh=20Fildadut?= <fildadut@reuh.eu>
Date: Thu, 4 Jan 2024 19:16:22 +0100
Subject: [PATCH] Improve translation template generation, only add the source
 context if needed

---
 anselme/ast/Function.lua      | 12 ++---
 anselme/ast/List.lua          |  2 +-
 anselme/ast/Struct.lua        |  7 ++-
 anselme/ast/Translatable.lua  |  9 ++--
 anselme/ast/Tuple.lua         | 10 +++-
 anselme/ast/Typed.lua         |  5 +-
 anselme/ast/abstract/Node.lua | 92 ++++++++++++++++++++++++++++++-----
 anselme/stdlib/persist.lua    |  5 +-
 anselme/stdlib/text.lua       |  4 +-
 ideas.md                      |  4 --
 10 files changed, 118 insertions(+), 32 deletions(-)

diff --git a/anselme/ast/Function.lua b/anselme/ast/Function.lua
index 5294ea2..7d2fba0 100644
--- a/anselme/ast/Function.lua
+++ b/anselme/ast/Function.lua
@@ -2,16 +2,16 @@
 
 local ast = require("anselme.ast")
 local Overloadable = ast.abstract.Overloadable
-local ReturnBoundary, Environment
+local ReturnBoundary, Environment, Identifier, Symbol
 
 local operator_priority = require("anselme.common").operator_priority
 
 local resume_manager, calling_environment_manager
 
 local function list_cache_upvalues(v, state, list, scope)
-	if ast.Identifier:is(v) then
+	if Identifier:is(v) then
 		list[v.name] = scope:precache(state, v)
-	elseif ast.Symbol:is(v) then
+	elseif Symbol:is(v) then
 		list[v.string] = scope:precache(state, v:to_identifier())
 	end
 	v:traverse(list_cache_upvalues, state, list, scope)
@@ -66,8 +66,8 @@ Function = Overloadable {
 		-- list & cache upvalues so they aren't affected by future redefinition in a parent scope
 		local upvalues = {}
 		self.expression:traverse(list_cache_upvalues, state, upvalues, scope)
-		if scope:defined(state, ast.Identifier:new("_")) then
-			scope:get(state, ast.Identifier:new("_")):traverse(list_cache_upvalues, state, upvalues, scope)
+		if scope:defined(state, Identifier:new("_")) then
+			scope:get(state, Identifier:new("_")):traverse(list_cache_upvalues, state, upvalues, scope)
 		end
 
 		return Function:new(self.parameters:eval(state), self.expression, scope, upvalues)
@@ -143,7 +143,7 @@ Function = Overloadable {
 }
 
 package.loaded[...] = Function
-ReturnBoundary, Environment = ast.ReturnBoundary, ast.Environment
+ReturnBoundary, Environment, Identifier, Symbol = ast.ReturnBoundary, ast.Environment, ast.Identifier, ast.Symbol
 
 resume_manager = require("anselme.state.resume_manager")
 calling_environment_manager = require("anselme.state.calling_environment_manager")
diff --git a/anselme/ast/List.lua b/anselme/ast/List.lua
index 7608015..315c36d 100644
--- a/anselme/ast/List.lua
+++ b/anselme/ast/List.lua
@@ -45,7 +45,7 @@ List = ast.abstract.Runtime {
 		return self.branched:get(state):len()
 	end,
 	iter = function(self, state)
-		return ipairs(self.branched:get(state).list)
+		return self.branched:get(state):iter()
 	end,
 	get = function(self, state, index)
 		local list = self.branched:get(state)
diff --git a/anselme/ast/Struct.lua b/anselme/ast/Struct.lua
index a94fcc9..7680156 100644
--- a/anselme/ast/Struct.lua
+++ b/anselme/ast/Struct.lua
@@ -44,7 +44,12 @@ Struct = ast.abstract.Runtime {
 		self.table = {}
 	end,
 	set = function(self, key, value) -- only for construction
-		self.table[key:hash()] = { key, value }
+		local hash = key:hash()
+		if Nil:is(value) then
+			self.table[hash] = nil
+		else
+			self.table[hash] = { key, value }
+		end
 	end,
 	include = function(self, other) -- only for construction
 		for _, e in pairs(other.table) do
diff --git a/anselme/ast/Translatable.lua b/anselme/ast/Translatable.lua
index 55744e8..eca852c 100644
--- a/anselme/ast/Translatable.lua
+++ b/anselme/ast/Translatable.lua
@@ -1,5 +1,5 @@
 local ast = require("anselme.ast")
-local TextInterpolation, String
+local TextInterpolation, String, Struct
 
 local operator_priority = require("anselme.common").operator_priority
 
@@ -10,11 +10,14 @@ local Translatable = ast.abstract.Node {
 	hide_in_stacktrace = true,
 
 	expression = nil,
+	context = nil, -- struct
 
 	init = function(self, expression)
 		self.expression = expression
-		self.context = ast.Struct:new()
+		self.context = Struct:new()
 		self.context:set(String:new("source"), String:new(self.expression.source))
+		self.context:set(String:new("file"), String:new(self.expression.source:match("^([^%:]*)")))
+		-- TODO: add parent script/function to context
 	end,
 
 	_format = function(self, ...)
@@ -46,7 +49,7 @@ local Translatable = ast.abstract.Node {
 }
 
 package.loaded[...] = Translatable
-TextInterpolation, String = ast.TextInterpolation, ast.String
+TextInterpolation, String, Struct = ast.TextInterpolation, ast.String, ast.Struct
 
 translation_manager = require("anselme.state.translation_manager")
 
diff --git a/anselme/ast/Tuple.lua b/anselme/ast/Tuple.lua
index 2d7c3c8..0fcf905 100644
--- a/anselme/ast/Tuple.lua
+++ b/anselme/ast/Tuple.lua
@@ -61,9 +61,17 @@ Tuple = ast.abstract.Node {
 		if index > #self.list or index == 0 then error("tuple index out of bounds", 0) end
 		return self.list[index]
 	end,
+	set = function(self, index, value)
+		if index < 0 then index = #self.list + 1 + index end
+		if index > #self.list or index == 0 then error("tuple index out of bounds", 0) end
+		self.list[index] = value
+	end,
 	len =  function(self)
 		return #self.list
-	end
+	end,
+	iter = function(self)
+		return ipairs(self.list)
+	end,
 }
 
 return Tuple
diff --git a/anselme/ast/Typed.lua b/anselme/ast/Typed.lua
index 20074a8..4652641 100644
--- a/anselme/ast/Typed.lua
+++ b/anselme/ast/Typed.lua
@@ -4,6 +4,8 @@ local operator_priority = require("anselme.common").operator_priority
 
 local format_identifier
 
+local ArgumentTuple
+
 local Typed
 Typed = ast.abstract.Runtime {
 	type = "typed",
@@ -20,7 +22,7 @@ Typed = ast.abstract.Runtime {
 		-- try custom format
 		if state and state.scope:defined(format_identifier) then
 			local custom_format = format_identifier:eval(state)
-			local args = ast.ArgumentTuple:new(self)
+			local args = ArgumentTuple:new(self)
 			local fn, d_args = custom_format:dispatch(state, args)
 			if fn then
 				return custom_format:call(state, d_args):format(state, prio, ...)
@@ -37,5 +39,6 @@ Typed = ast.abstract.Runtime {
 
 package.loaded[...] = Typed
 format_identifier = ast.Identifier:new("format")
+ArgumentTuple = ast.ArgumentTuple
 
 return Typed
diff --git a/anselme/ast/abstract/Node.lua b/anselme/ast/abstract/Node.lua
index f4204da..79dceeb 100644
--- a/anselme/ast/abstract/Node.lua
+++ b/anselme/ast/abstract/Node.lua
@@ -2,6 +2,7 @@ local class = require("anselme.lib.class")
 local fmt = require("anselme.common").fmt
 local binser = require("anselme.lib.binser")
 local utf8 = utf8 or require("lua-utf8")
+local unpack = table.unpack or unpack
 
 -- NODES SHOULD BE IMMUTABLE AFTER CREATION IF POSSIBLE!
 -- i don't think i actually rely on this behavior for anything but it makes me feel better about life in general
@@ -13,7 +14,7 @@ local utf8 = utf8 or require("lua-utf8")
 
 local uuid = require("anselme.common").uuid
 
-local Call, Identifier, ArgumentTuple
+local Call, Identifier, ArgumentTuple, Struct, Tuple, String, Pair
 local resume_manager
 
 local custom_call_identifier
@@ -143,17 +144,86 @@ Node = class {
 
 	-- generate anselme code that can be used as a base for a translation file
 	-- will include every translatable element found in this node and its children
-	-- TODO: generate more stable context than source position, and only add necessery context to the tag
 	generate_translation_template = function(self)
-		local l = self:list_translatable()
-		local r = {}
-		for _, tr in ipairs(l) do
-			table.insert(r, "(("..tr.source.."))")
-			table.insert(r, Call:new(Identifier:new("_#_"), ArgumentTuple:new(tr.context, Identifier:new("_"))):format())
-			table.insert(r, "\t"..Call:new(Identifier:new("_->_"), ArgumentTuple:new(tr, tr)):format())
-			table.insert(r, "")
+		local mandatory_context = {"file"} -- will always be present in template
+		local discriminating_context = "source" -- only used when there may be conflicts
+
+		-- if there are several identical translations in the tuple, wrap them in a struct with the discriminating context
+		local function classify_discriminating(tr_tuple)
+			local r_tuple = Tuple:new()
+			local discovered = Struct:new()
+			for _, tr in tr_tuple:iter() do
+				if not discovered:has(tr) then
+					r_tuple:insert(tr)
+					discovered:set(tr, r_tuple:len())
+				else
+					local context_name = String:new(discriminating_context)
+					-- replace previousley discovered translation
+					local discovered_index = discovered:get(tr)
+					if discovered_index ~= -1 then
+						local discovered_tr = r_tuple:get(discovered_index)
+						local context = Pair:new(context_name, discovered_tr.context:get(context_name))
+						local discovered_strct = Struct:new()
+						discovered_strct:set(context, discovered_tr)
+						r_tuple:set(discovered_index, discovered_strct)
+						discovered:set(tr, -1)
+					end
+					-- add current translation
+					local context = Pair:new(context_name, tr.context:get(context_name))
+					local strct = Struct:new()
+					strct:set(context, tr)
+					r_tuple:insert(strct)
+				end
+			end
+			return r_tuple
 		end
-		return table.concat(r, "\n")
+
+		-- build a Struct{ [context1] = Struct{ [context2]} = Tuple[translatable, Struct { [context3] = translatable }, ...], ... }, ... }
+		local function classify(tr_tuple, context_i)
+			local r = Struct:new()
+			local context_name = String:new(mandatory_context[context_i])
+			for _, tr in tr_tuple:iter() do
+				local context = Pair:new(context_name, tr.context:get(context_name))
+				if not r:has(context) then
+					r:set(context, Tuple:new())
+				end
+				local context_tr_tuple = r:get(context)
+				context_tr_tuple:insert(tr)
+			end
+			for context, context_tr_tuple in r:iter() do
+				if context_i < #mandatory_context then
+					r:set(context, classify(context_tr_tuple, context_i+1))
+				elseif context_i == #mandatory_context then
+					r:set(context, classify_discriminating(context_tr_tuple))
+				end
+			end
+			return r
+		end
+
+		local classified = classify(Tuple:new(unpack(self:list_translatable())), 1)
+
+		local function build_str(tr, level)
+			level = level or 0
+			local indent = ("\t"):rep(level)
+			local r = {}
+			if Struct:is(tr) then
+				for context, context_tr in tr:iter() do
+					table.insert(r, indent..Call:new(Identifier:new("_#_"), ArgumentTuple:new(context, Identifier:new("_"))):format():gsub(" _$", ""))
+					table.insert(r, build_str(context_tr, level+1))
+				end
+			elseif Tuple:is(tr) then
+				for _, v in tr:iter() do
+					table.insert(r, build_str(v, level))
+					table.insert(r, "")
+				end
+			else
+				table.insert(r, indent.."/* "..tr.source.." */")
+				table.insert(r, indent..Call:new(Identifier:new("_->_"), ArgumentTuple:new(tr, tr)):format())
+			end
+			return table.concat(r, "\n")
+		end
+
+		return build_str(classified)
 	end,
 
 	-- call the node with the given arguments
@@ -332,7 +402,7 @@ Node = class {
 	-- Thus, any require here that may require other Nodes shall be done here. This method is called in anselme.lua after everything else is required.
 	_i_hate_cycles = function(self)
 		local ast = require("anselme.ast")
-		Call, Identifier, ArgumentTuple = ast.Call, ast.Identifier, ast.ArgumentTuple
+		Call, Identifier, ArgumentTuple, Struct, Tuple, String, Pair = ast.Call, ast.Identifier, ast.ArgumentTuple, ast.Struct, ast.Tuple, ast.String, ast.Pair
 		custom_call_identifier = Identifier:new("_!")
 
 		resume_manager = require("anselme.state.resume_manager")
diff --git a/anselme/stdlib/persist.lua b/anselme/stdlib/persist.lua
index 8582a8a..6ce9f2b 100644
--- a/anselme/stdlib/persist.lua
+++ b/anselme/stdlib/persist.lua
@@ -1,4 +1,5 @@
 local ast = require("anselme.ast")
+local Nil = ast.Nil
 
 local persistent_manager = require("anselme.state.persistent_manager")
 
@@ -13,7 +14,7 @@ return {
 		"persist", "(key, default) = value",
 		function(state, key, default, value)
 			persistent_manager:set(state, key, value)
-			return ast.Nil:new()
+			return Nil:new()
 		end
 	},
 	{
@@ -26,7 +27,7 @@ return {
 		"persist", "(key) = value",
 		function(state, key, value)
 			persistent_manager:set(state, key, value)
-			return ast.Nil:new()
+			return Nil:new()
 		end
 	},
 }
diff --git a/anselme/stdlib/text.lua b/anselme/stdlib/text.lua
index 1ee0325..f73a472 100644
--- a/anselme/stdlib/text.lua
+++ b/anselme/stdlib/text.lua
@@ -1,5 +1,5 @@
 local ast = require("anselme.ast")
-local Nil, Choice, PartialScope, ArgumentTuple = ast.Nil, ast.Choice, ast.PartialScope, ast.ArgumentTuple
+local Nil, Choice, PartialScope, ArgumentTuple, Identifier = ast.Nil, ast.Choice, ast.PartialScope, ast.ArgumentTuple, ast.Identifier
 
 local event_manager = require("anselme.state.event_manager")
 local translation_manager = require("anselme.state.translation_manager")
@@ -34,7 +34,7 @@ return {
 	{
 		"_->_", "(original::is(\"quote\"), translated::is(\"quote\"))",
 		function(state, original, translated)
-			local exp = PartialScope:preserve(state, translated.expression, ast.Identifier:new("_"))
+			local exp = PartialScope:preserve(state, translated.expression, Identifier:new("_"))
 			translation_manager:set(state, tag_manager:get(state), original.expression, exp)
 			return Nil:new()
 		end
diff --git a/ideas.md b/ideas.md
index 279c77a..307f557 100644
--- a/ideas.md
+++ b/ideas.md
@@ -14,10 +14,6 @@ Translation.
 
 Translation model for stdlib: ?
 
-For translation context/disambiguation:
-* text+line+file - OK
-* func/var hierarchy?
-
 Do some more fancy scope work to allow the translation to access variables defined in the translation file?
 
 ---