[internal] do block parsing at the same time as expression parsing

anselme/ast/Call.lua

@@ -16,6 +16,7 @@ local Call
 
 Call = ast.abstract.Node {
 	type = "call",
+	explicit = true, -- false for implicit calls
 
 	func = nil,
 	arguments = nil, -- ArgumentTuple

anselme/parser/Source.lua

@@ -15,12 +15,18 @@ Source = class {
 	increment = function(self, n, ...)
 		self.position = self.position + n
 	end,
+	increment_line = function(self, n, ...)
+		self.line = self.line + n
+	end,
 	count = function(self, capture, ...)
-		self:increment(utf8.len(capture))
-		return capture, ...
+		return capture, self:consume(capture, ...)
 	end,
 	consume = function(self, capture, ...)
-		self:increment(utf8.len(capture))
+		for _ in capture:gmatch(".-\n") do
+			self.position = 1
+			self:increment_line(1)
+		end
+		self:increment(utf8.len(capture:match("[^\n]*$")))
 		return ...
 	end,
 

anselme/parser/code_to_tree.lua

@@ -1,67 +0,0 @@
---- transform raw code string into a nested tree of lines
-
-local utf8 = utf8 or require("lua-utf8")
-
-local Source = require("anselme.parser.Source")
-
-local function indented_to_tree(indented)
-	local tree = {}
-	local current_parent = tree
-	local current_level = 0
-	local last_line_empty = nil
-
-	for _, l in ipairs(indented) do
-		-- indentation of empty line is determined using the next line
-		-- (consecutive empty lines are merged into one)
-		if l.content == "" then
-			last_line_empty = l
-		else
-			-- raise indentation
-			if l.level > current_level then
-				if #current_parent == 0 then -- can't add children to nil
-					error(("invalid indentation; at %s"):format(l.source))
-				end
-				current_parent = current_parent[#current_parent]
-				current_level = l.level
-			-- lower indentation
-			elseif l.level < current_level then
-				current_parent = tree
-				current_level = 0
-				while current_level < l.level do -- find correct level starting back from the root
-					current_parent = current_parent[#current_parent]
-					current_level = current_parent[1].level
-				end
-				if current_level ~= l.level then
-					error(("invalid indentation; at %s"):format(l.source))
-				end
-			end
-			-- add line
-			if last_line_empty then
-				last_line_empty.level = current_level
-				table.insert(current_parent, last_line_empty)
-				last_line_empty = nil
-			end
-			table.insert(current_parent, l)
-		end
-	end
-
-	return tree
-end
-
-local function code_to_indented(code, source_name)
-	local indented = {}
-
-	local i = 1
-	for line in (code.."\n"):gmatch("(.-)\n") do
-		local indent, rem = line:match("^(%s*)(.-)$")
-		local indent_len = utf8.len(indent)
-		table.insert(indented, { level = indent_len, content = rem, source = Source:new(source_name, i, 1+indent_len) })
-		i = i + 1
-	end
-
-	return indented
-end
-
-return function(code, source_name)
-	return indented_to_tree(code_to_indented(code, source_name or "?"))
-end

anselme/parser/expression/block.lua

@@ -0,0 +1,70 @@
+local expression_to_ast = require("anselme.parser.expression.to_ast")
+
+local ast = require("anselme.ast")
+local PartialScope, Block, Flush, Call, Identifier = ast.PartialScope, ast.Block, ast.Flush, ast.Call, ast.Identifier
+
+local function block(source, str)
+	local start_source = source:clone()
+
+	if not str:match("^\n") then
+		str = "\n"..str
+		source:increment_line(-1)
+	end
+
+	local levels = { { indentation = utf8.len(str:match("^\n([ \t]*)")), block = Block:new() } }
+	local current_level = levels[#levels]
+
+	local rem = str
+	local last_line_empty
+	while rem:match("^\n") do
+		local line = source:consume(rem:match("^(\n)(.*)$"))
+		local new_indentation = utf8.len(line:match("^([ \t]*)"))
+		-- indentation of empty line is determined using the next line
+		-- (consecutive empty lines are merged into one)
+		if line:match("^\n") then
+			rem = line
+			last_line_empty = true
+		elseif line:match("[^%s]") then
+			-- raise indentation
+			if new_indentation > current_level.indentation then
+				local child_block = Block:new()
+				local cur_exps = current_level.block.expressions
+				cur_exps[#cur_exps] = PartialScope:attach_block(cur_exps[#cur_exps], child_block):set_source(source)
+				table.insert(levels, { indentation = new_indentation, block = child_block })
+				current_level = levels[#levels]
+			-- lower indentation
+			elseif new_indentation < current_level.indentation then
+				while new_indentation < current_level.indentation do
+					table.remove(levels)
+					current_level = levels[#levels]
+				end
+				if new_indentation ~= current_level.indentation then
+					error(("invalid indentation; at %s"):format(source))
+				end
+			end
+
+			-- parse line
+			local s, exp
+			s, exp, rem = pcall(expression_to_ast, source, line)
+			if not s then error(("invalid expression in block: %s"):format(exp), 0) end
+
+			-- single implicit _: line was empty (e.g. single comment in the line)
+			if Call:is(exp) and not exp.explicit and Identifier:is(exp.func) and exp.func.name == "_" then
+				exp = Flush:new()
+			end
+
+			-- add line
+			if last_line_empty then
+				current_level.block:add(Flush:new())
+				last_line_empty = nil
+			end
+			current_level.block:add(exp)
+		else -- end-of-file
+			rem = ""
+		end
+	end
+
+	return levels[1].block:set_source(start_source), rem
+end
+
+return block

anselme/parser/expression/comment.lua

@@ -11,7 +11,7 @@ comment = primary {
 		local content_list = {}
 		while not rem:match("^%*%/") do
 			local content
-			content, rem = rem:match("^([^%/%*]*)(.-)$")
+			content, rem = rem:match("^([^\n%/%*]*)(.-)$")
 
 			-- cut the text prematurely at limit_pattern if relevant
 			if limit_pattern and content:match(limit_pattern) then
@@ -32,6 +32,10 @@ comment = primary {
 				table.insert(content_list, "/*")
 				table.insert(content_list, subcomment)
 				table.insert(content_list, "*/")
+			-- consumed everything until end-of-line/file, close your eyes and imagine the text has been closed
+			elseif rem:match("^\n") or not rem:match("[^%s]") then
+				rem = "*/" .. rem
+				source:increment(-2)
 			-- no end token after the comment
 			elseif not rem:match("^%*%/") then
 				-- single * or /, keep on commentin'
@@ -39,12 +43,9 @@ comment = primary {
 					local s
 					s, rem = source:count(rem:match("^([%*%/])(.-)$"))
 					table.insert(content_list, s)
-				-- anything other than end-of-line
-				elseif rem:match("[^%s]") then
-					error(("unexpected %q at end of comment"):format(rem), 0)
-				-- consumed everything until end-of-line, close your eyes and imagine the text has been closed
+				-- anything else
 				else
-					rem = rem .. "*/"
+					error(("unexpected %q at end of comment"):format(rem:match("^[^\n]*")), 0)
 				end
 			end
 		end

anselme/parser/expression/contextual/function_parameter.lua

@@ -21,16 +21,16 @@ return primary {
 
 		-- value check
 		local value_check
-		if rem:match("^%s*::") then
-			local scheck = source:consume(rem:match("^(%s*::%s*)(.*)$"))
+		if rem:match("^[ \t]*::") then
+			local scheck = source:consume(rem:match("^([ \t]*::[ \t]*)(.*)$"))
 			value_check, rem = expression_to_ast(source, scheck, limit_pattern, value_check_priority)
 		end
 
 		-- default value
 		local default
 		if not no_default_value then
-			if rem:match("^%s*=") then
-				local sdefault = source:consume(rem:match("^(%s*=%s*)(.*)$"))
+			if rem:match("^[ \t]*=") then
+				local sdefault = source:consume(rem:match("^([ \t]*=[ \t]*)(.*)$"))
 				default, rem = expression_to_ast(source, sdefault, limit_pattern, assignment_priority)
 			end
 		end

anselme/parser/expression/contextual/parameter_tuple.lua

@@ -16,27 +16,27 @@ return primary {
 
 		-- i would LOVE to reuse the regular list parsing code for this, but unfortunately the list parsing code
 		-- itself depends on this and expect this to be available quite early, and it's ANNOYING
-		while not rem:match("^%s*%)") do
+		while not rem:match("^[ \t]*%)") do
 			-- parameter
 			local func_param
 			func_param, rem = function_parameter:expect(source, rem, limit_pattern)
 
 			-- next! comma separator
-			if not rem:match("^%s*%)") then
-				if not rem:match("^%s*,") then
-					error(("unexpected %q at end of argument list"):format(rem), 0)
+			if not rem:match("^[ \t]*%)") then
+				if not rem:match("^[ \t]*,") then
+					error(("unexpected %q at end of argument list"):format(rem:match("^[^\n]*")), 0)
 				end
-				rem = source:consume(rem:match("^(%s*,)(.*)$"))
+				rem = source:consume(rem:match("^([ \t]*,)(.*)$"))
 			end
 
 			-- add
 			parameters:insert(func_param)
 		end
-		rem = rem:match("^%s*%)(.*)$")
+		rem = rem:match("^[ \t]*%)(.*)$")
 
 		-- assigment param
-		if rem:match("^%s*=") then
-			rem = source:consume(rem:match("^(%s*=%s*)(.*)$"))
+		if rem:match("^[ \t]*=") then
+			rem = source:consume(rem:match("^([ \t]*=[ \t]*)(.*)$"))
 
 			local func_param
 			func_param, rem = function_parameter_no_default:expect(source, rem, limit_pattern)

anselme/parser/expression/primary/function_definition.lua

@@ -32,8 +32,8 @@ local function_parameter_maybe_parenthesis = function_parameter_no_default {
 
 			local exp, rem = function_parameter_no_default:parse(source, str, limit_pattern)
 
-			if not rem:match("^%s*%)") then error(("unexpected %q at end of parenthesis"):format(rem), 0) end
-			rem = source:consume(rem:match("^(%s*%))(.-)$"))
+			if not rem:match("^[ \t]*%)") then error(("unexpected %q at end of parenthesis"):format(rem:match("^[^\n]*")), 0) end
+			rem = source:consume(rem:match("^([ \t]*%))(.-)$"))
 
 			return exp, rem
 		else
@@ -49,7 +49,7 @@ local function_parameter_maybe_parenthesis = function_parameter_no_default {
 local function search_prefix_signature(modifiers, source, str, limit_pattern)
 	for _, pfx in ipairs(prefixes) do
 		local prefix = pfx[1]
-		local prefix_pattern = "%s*"..escape(prefix).."%s*"
+		local prefix_pattern = "[ \t]*"..escape(prefix).."[ \t]*"
 		if str:match("^"..prefix_pattern) then
 			-- operator name
 			local rem = source:consume(str:match("^("..prefix_pattern..")(.*)$"))
@@ -82,7 +82,7 @@ local function search_infix_signature(modifiers, source, str, limit_pattern)
 
 		for _, ifx in ipairs(infixes) do
 			local infix = ifx[1]
-			local infix_pattern = "%s*"..escape(infix).."%s*"
+			local infix_pattern = "[ \t]*"..escape(infix).."[ \t]*"
 			if rem:match("^"..infix_pattern) then
 				-- operator name
 				rem = src:consume(rem:match("^("..infix_pattern..")(.*)$"))
@@ -119,7 +119,7 @@ local function search_suffix_signature(modifiers, source, str, limit_pattern)
 
 		for _, sfx in ipairs(suffixes) do
 			local suffix = sfx[1]
-			local suffix_pattern = "%s*"..escape(suffix).."%s*"
+			local suffix_pattern = "[ \t]*"..escape(suffix).."[ \t]*"
 			if rem:match("^"..suffix_pattern) then
 				-- operator name
 				rem = src:count(rem:match("^("..suffix_pattern..")(.*)$"))

anselme/parser/expression/primary/identifier.lua

@@ -2,8 +2,8 @@ local primary = require("anselme.parser.expression.primary.primary")
 
 local Identifier = require("anselme.ast.Identifier")
 
-local disallowed_set = (".~`^+-=<>/[]*{}|\\_!?,;:()\"@&$#%"):gsub("[^%w]", "%%%1")
-local identifier_pattern = "%s*[^0-9%s'"..disallowed_set.."][^"..disallowed_set.."]*"
+local disallowed_set = ("\n.~`^+-=<>/[]*{}|\\_!?,;:()\"@&$#%"):gsub("[^%w]", "%%%1")
+local identifier_pattern = "[ \t]*[^0-9%s'"..disallowed_set.."][^"..disallowed_set.."]*"
 
 local common = require("anselme.common")
 local trim, escape = common.trim, common.escape
@@ -17,7 +17,7 @@ for _, suffix in ipairs(regular_operators.suffixes) do table.insert(operators, "
 
 -- all valid identifier patterns
 local identifier_patterns = { identifier_pattern }
-for _, operator in ipairs(operators) do table.insert(identifier_patterns, "%s*"..escape(operator)) end
+for _, operator in ipairs(operators) do table.insert(identifier_patterns, "[ \t]*"..escape(operator)) end
 
 return primary {
 	match = function(self, str)

anselme/parser/expression/primary/implicit_block_identifier.lua

@@ -0,0 +1,17 @@
+local primary = require("anselme.parser.expression.primary.primary")
+
+local ast = require("anselme.ast")
+local Identifier, Call, ArgumentTuple = ast.Identifier, ast.Call, ast.ArgumentTuple
+
+return primary {
+	match = function(self, str)
+		return str:match("^\n")
+	end,
+
+	parse = function(self, source, str)
+		-- implicit _, do not consume the newline
+		local r = Call:new(Identifier:new("_"), ArgumentTuple:new()):set_source(source)
+		r.explicit = false
+		return r, str
+	end
+}

anselme/parser/expression/primary/init.lua

@@ -1,5 +1,7 @@
 --- try to parse a primary expression
 
+local comment = require("anselme.parser.expression.comment")
+
 local function r(name)
 	return require("anselme.parser.expression.primary."..name), nil
 end
@@ -14,6 +16,7 @@ local primaries = {
 	r("identifier"),
 	r("anchor"),
 	r("block_identifier"),
+	r("implicit_block_identifier"),
 	r("tuple"),
 	r("struct"),
 
@@ -31,6 +34,11 @@ return {
 	-- returns exp, rem if expression found
 	-- returns nil if no expression found
 	search = function(self, source, str, limit_pattern)
+		str = source:consume(str:match("^([ \t]*)(.*)$"))
+		-- if there is a comment, restart the parsing after the comment ends
+		local c, c_rem = comment:search(source, str, limit_pattern)
+		if c then return self:search(source, c_rem, limit_pattern) end
+		-- search primary
 		for _, primary in ipairs(primaries) do
 			local exp, rem = primary:search(source, str, limit_pattern)
 			if exp then return exp, rem end

anselme/parser/expression/primary/parenthesis.lua

@@ -22,7 +22,7 @@ return primary {
 			local s
 			s, exp, rem = pcall(expression_to_ast, source, rem, "%)")
 			if not s then error("invalid expression inside parentheses: "..exp, 0) end
-			if not rem:match("^%s*%)") then error(("unexpected %q at end of parenthesis"):format(rem), 0) end
+			if not rem:match("^%s*%)") then error(("unexpected %q at end of parenthesis"):format(rem:match("^[^\n]*")), 0) end
 		end
 		rem = source:consume(rem:match("^(%s*%))(.*)$"))
 

anselme/parser/expression/primary/string.lua

@@ -37,7 +37,7 @@ return primary {
 		while not rem:match("^"..stop_pattern) do
 			local text_source = source:clone()
 			local text
-			text, rem = rem:match("^([^%{%\\"..stop_pattern.."]*)(.-)$") -- get all text until something potentially happens
+			text, rem = rem:match("^([^\n%{%\\"..stop_pattern.."]*)(.-)$") -- get all text until something potentially happens
 
 			-- cut the text prematurely at limit_pattern if relevant
 			if self.allow_implicit_stop and limit_pattern and text:match(limit_pattern) then
@@ -50,23 +50,25 @@ return primary {
 
 			interpolation:insert(String:new(text):set_source(text_source))
 
+			-- interpolated expression
 			if rem:match("^%{") then
 				local ok, exp
 				ok, exp, rem = pcall(expression_to_ast, source, source:consume(rem:match("^(%{)(.*)$")), "%}")
 				if not ok then error("invalid expression inside interpolation: "..exp, 0) end
-				if not rem:match("^%s*%}") then error(("unexpected %q at end of interpolation"):format(rem), 0) end
-				rem = source:consume(rem:match("^(%s*%})(.*)$"))
+				if not rem:match("^[ \t]*%}") then error(("unexpected %q at end of interpolation"):format(rem:match("^[^\n]*")), 0) end
+				rem = source:consume(rem:match("^([ \t]*%})(.*)$"))
 				interpolation:insert(exp)
+			-- escape sequence
 			elseif rem:match("^\\") then
 				text, rem = source:consume(rem:match("^(\\(.))(.*)$"))
 				interpolation:insert(String:new(escape_code[text] or text))
+			-- consumed everything until end-of-line/file, implicit stop allowed, close your eyes and imagine the text has been closed
+			elseif self.allow_implicit_stop and (rem:match("^\n") or not rem:match("[^%s]")) then
+				rem = self.stop_char .. rem
+				source:increment(-1)
+			-- no end token after the comment
 			elseif not rem:match("^"..stop_pattern) then
-				if not self.allow_implicit_stop or rem:match("[^%s]") then
-					error(("unexpected %q at end of "..self.type):format(rem), 0)
-				-- consumed everything until end-of-line, implicit stop allowed, close your eyes and imagine the text has been closed
-				else
-					rem = rem .. self.stop_char
-				end
+				error(("unexpected %q at end of "..self.type):format(rem:match("^[^\n]*")), 0)
 			end
 		end
 		rem = source:consume(rem:match("^("..stop_pattern..")(.*)$"))

anselme/parser/expression/primary/text.lua

@@ -5,7 +5,7 @@ local TextInterpolation, Translatable = ast.TextInterpolation, ast.Translatable
 
 return string {
 	type = "text",
-	start_pattern = "|%s?",
+	start_pattern = "|[ \t]?",
 	stop_char = "|",
 	allow_implicit_stop = true,
 	interpolation = TextInterpolation,
@@ -16,7 +16,7 @@ return string {
 
 		-- remove terminal space
 		local last = interpolation.list[#interpolation.list]
-		if ast.String:is(last) then last.string = last.string:gsub("%s$", "") end
+		if ast.String:is(last) then last.string = last.string:gsub("[ \t]$", "") end
 
 		return Translatable:new(interpolation):set_source(start_source), rem
 	end

anselme/parser/expression/primary/tuple.lua

@@ -22,18 +22,18 @@ return primary {
 		local end_match = escape(end_char)
 
 		local l
-		if not rem:match("^%s*"..end_match) then
+		if not rem:match("^[ \t]*"..end_match) then
 			local s
-			s, l, rem = pcall(expression_to_ast, source, rem, end_match)
+			s, l, rem = pcall(expression_to_ast, source, rem, end_match, nil)
 			if not s then error("invalid expression in list: "..l, 0) end
 		end
 
 		if not Tuple:is(l) or l.explicit then l = Tuple:new(l) end -- single or no element
 
-		if not rem:match("^%s*"..end_match) then
-			error(("unexpected %q at end of list"):format(rem), 0)
+		if not rem:match("^[ \t]*"..end_match) then
+			error(("unexpected %q at end of list"):format(rem:match("^[^\n]*")), 0)
 		end
-		rem = source:consume(rem:match("^(%s*"..end_match..")(.*)$"))
+		rem = source:consume(rem:match("^([ \t]*"..end_match..")(.*)$"))
 
 		l.explicit = true
 		return l:set_source(start_source), rem

anselme/parser/expression/secondary/infix/call.lua

@@ -14,7 +14,7 @@ return infix {
 
 	match = function(self, str, current_priority, primary)
 		local escaped = escape(self.operator)
-		return self.priority > current_priority and str:match("^"..escaped) and identifier:match(str:match("^"..escaped.."%s*(.-)$"))
+		return self.priority > current_priority and str:match("^"..escaped) and identifier:match(str:match("^"..escaped.."[ \t]*(.-)$"))
 	end,
 
 	build_ast = function(self, left, right)

anselme/parser/expression/secondary/infix/implicit_multiplication.lua

@@ -18,6 +18,8 @@ return infix {
 	parse = function(self, source, str, limit_pattern, current_priority, primary)
 		local start_source = source:clone()
 		local right, rem = identifier:parse(source, str, limit_pattern)
-		return Call:new(Identifier:new(self.identifier), ArgumentTuple:new(primary, right)):set_source(start_source), rem
+		local r = Call:new(Identifier:new(self.identifier), ArgumentTuple:new(primary, right)):set_source(start_source)
+		r.explicit = false
+		return r, rem
 	end
 }

anselme/parser/expression/secondary/infix/infix.lua

@@ -23,7 +23,7 @@ return secondary {
 
 		local sright = source:consume(str:match("^("..escaped..")(.*)$"))
 		local s, right, rem = pcall(expression_to_ast, source, sright, limit_pattern, self.priority)
-		if not s then error(("invalid expression after binary operator %q: %s"):format(self.operator, right), 0) end
+		if not s then error(("invalid expression after infix operator %q: %s"):format(self.operator, right), 0) end
 
 		return self:build_ast(primary, right):set_source(start_source), rem
 	end,

anselme/parser/expression/secondary/infix/tuple.lua

@@ -20,8 +20,8 @@ return infix {
 
 		local escaped = escape(self.operator)
 		local rem = str
-		while rem:match("^%s*"..escaped) do
-			rem = source:consume(rem:match("^(%s*"..escaped..")(.*)$"))
+		while rem:match("^[ \t]*"..escaped) do
+			rem = source:consume(rem:match("^([ \t]*"..escaped..")(.*)$"))
 
 			local s, right
 			s, right, rem = pcall(expression_to_ast, source, rem, limit_pattern, self.priority)

anselme/parser/expression/secondary/init.lua

@@ -1,11 +1,13 @@
 --- try to parse a secondary expression
 
+local comment = require("anselme.parser.expression.comment")
+
 local function r(name)
 	return require("anselme.parser.expression.secondary."..name), nil
 end
 
 local secondaries = {
-	-- binary infix operators
+	-- binary infix operators,
 	r("infix.semicolon"),
 	r("infix.tuple"),
 	r("infix.tag"),
@@ -49,8 +51,16 @@ end
 return {
 	-- returns exp, rem if expression found
 	-- returns nil if no expression found
-	-- returns nil, err if error
 	search = function(self, source, str, limit_pattern, current_priority, primary)
+		str = source:consume(str:match("^([ \t]*)(.*)$"))
+		-- if there is a comment, restart the parsing after the comment ends
+		local c, c_rem = comment:search(source, str, limit_pattern)
+		if c then
+			local ce, ce_rem = self:search(source, c_rem, limit_pattern, current_priority, primary)
+			if ce then return ce, ce_rem
+			else return primary, c_rem end -- noop
+		end
+		-- search secondary
 		for _, secondary in ipairs(secondaries) do
 			local exp, rem = secondary:search(source, str, limit_pattern, current_priority, primary)
 			if exp then return exp, rem end

anselme/parser/expression/secondary/suffix/call.lua

@@ -27,7 +27,7 @@ return secondary {
 			exp, rem = parenthesis:parse(source, str, limit_pattern)
 
 			if Nil:is(exp) then
-				if str:match("^%(%s*%(%s*%)%s*%)") then -- special case: single nil argument
+				if str:match("^%([ \t]*%([ \t]*%)[ \t]*%)") then -- special case: single nil argument
 					exp = Tuple:new(Nil:new())
 				else -- no arguments
 					exp = Tuple:new()

anselme/parser/expression/to_ast.lua

@@ -2,16 +2,10 @@
 
 local primary, secondary
 
-local comment = require("anselme.parser.expression.comment")
-
 -- parse an expression, starting from a secondary element operating on operating_on_primary
 -- returns expr, remaining
 local function from_secondary(source, s, limit_pattern, current_priority, operating_on_primary)
-	s = source:consume(s:match("^(%s*)(.*)$"))
 	current_priority = current_priority or 0
-	-- if there is a comment, restart the parsing after the comment ends
-	local c, c_rem = comment:search(source, s, limit_pattern)
-	if c then return from_secondary(source, c_rem, limit_pattern, current_priority, operating_on_primary) end
 	-- secondary elements
 	local exp, rem = secondary:search(source, s, limit_pattern, current_priority, operating_on_primary)
 	if exp then return from_secondary(source, rem, limit_pattern, current_priority, exp) end
@@ -24,18 +18,13 @@ end
 -- limit_pattern: set to a string pattern that will trigger the end of elements that would otherwise consume everything until end-of-line (pattern is not consumed)
 -- fallback_exp: if no primary expression can be found, will return this instead. Used to avoid raising an error where an empty or comment-only expression is allowed.
 -- return expr, remaining
-local function expression_to_ast(source, s, limit_pattern, current_priority, fallback_exp)
-	s = source:consume(s:match("^(%s*)(.*)$"))
+local function expression_to_ast(source, s, limit_pattern, current_priority)
 	current_priority = current_priority or 0
-	-- if there is a comment, restart the parsing after the comment ends
-	local c, c_rem = comment:search(source, s, limit_pattern)
-	if c then return expression_to_ast(source, c_rem, limit_pattern, current_priority, fallback_exp) end
 	-- primary elements
 	local exp, rem = primary:search(source, s, limit_pattern)
 	if exp then return from_secondary(source, rem, limit_pattern, current_priority, exp) end
 	-- no valid primary expression
-	if fallback_exp then return fallback_exp, s end
-	error(("no valid expression before %q"):format(s), 0)
+	error(("no valid expression after %q"):format(s), 0)
 end
 
 package.loaded[...] = expression_to_ast
@@ -44,7 +33,7 @@ primary = require("anselme.parser.expression.primary")
 secondary = require("anselme.parser.expression.secondary")
 
 -- return expr, remaining
-return function(source, s, limit_pattern, current_priority, operating_on_primary, fallback_exp)
+return function(source, s, limit_pattern, current_priority, operating_on_primary)
 	if operating_on_primary then return from_secondary(source, s, limit_pattern, current_priority, operating_on_primary)
-	else return expression_to_ast(source, s, limit_pattern, current_priority, fallback_exp) end
+	else return expression_to_ast(source, s, limit_pattern, current_priority) end
 end

anselme/parser/init.lua

@@ -1,11 +1,15 @@
-local code_to_tree = require("anselme.parser.code_to_tree")
-local tree_to_ast = require("anselme.parser.tree_to_ast")
+local block = require("anselme.parser.expression.block")
+local Source = require("anselme.parser.Source")
+
+local function expect_end(exp, rem)
+	if rem:match("[^%s]") then
+		error(("expected end of expression before %q"):format(rem))
+	end
+	return exp
+end
 
 -- parse code (string) with the associated source (Source)
 -- the returned AST tree is stateless and can be stored/evaluated/etc as you please
 return function(code, source)
-	local tree = code_to_tree(code, source)
-	local block = tree_to_ast(tree)
-
-	return block
+	return expect_end(block(Source:new(source, 1, 1), code))
 end

anselme/parser/tree_to_ast.lua

@@ -1,52 +0,0 @@
---- transform a tree of lines into raw AST
-
-local tree_to_block
-
-local ast = require("anselme.ast")
-local Block, Flush, PartialScope
-
-local expression_to_ast = require("anselme.parser.expression.to_ast")
-
--- wrapper for expression_to_ast to check that there is no crap remaining after the expression has been parsed
--- return AST
-local function expect_end(exp, rem)
-	if rem:match("[^%s]") then
-		error(("expected end of expression before %q"):format(rem))
-	end
-	return exp
-end
-local function expect_end_block(exp, rem)
-	if rem:match("[^%s]") and not rem:match("^ ?_$") then
-		error(("expected end of expression before %q"):format(rem))
-	end
-	return exp
-end
-
--- return AST
-local function line_to_expression(content, tree)
-	if #tree > 0 then
-		local child_block = tree_to_block(tree)
-		return PartialScope:attach_block(expect_end_block(expression_to_ast(tree.source:clone(), content.." _", " _$")), child_block):set_source(tree.source)
-	else
-		return expect_end(expression_to_ast(tree.source:clone(), content, nil, nil, nil, Flush:new())):set_source(tree.source)
-	end
-end
-
--- return AST (Block)
-tree_to_block = function(tree)
-	local block = Block:new()
-
-	for _, l in ipairs(tree) do
-		local s, expression = pcall(line_to_expression, l.content, l)
-		if not s then error(("%s; at %s"):format(expression, l.source), 0) end
-
-		block:add(expression)
-	end
-
-	return block
-end
-
-package.loaded[...] = tree_to_block
-Block, Flush, PartialScope = ast.Block, ast.Flush, ast.PartialScope
-
-return tree_to_block

test/results/checkpoint merging mutable value.ans

@@ -10,7 +10,7 @@
 --- error ---
 cancel merge
 	↳ from test/tests/checkpoint merging mutable value.ans:24:6 in call: error("cancel merge")
-	↳ from ? in block: :l = *[1, 2]…
+	↳ from test/tests/checkpoint merging mutable value.ans:1:1 in block: :l = *[1, 2]…
 --# post run check #--
 --- text ---
 | {}"1,2,3,4: " {}"*[1, 2, 3, 4]" {}"" |

test/results/checkpoint merging variable.ans

@@ -10,7 +10,7 @@
 --- error ---
 cancel merge
 	↳ from test/tests/checkpoint merging variable.ans:24:6 in call: error("cancel merge")
-	↳ from ? in block: :l = 1…
+	↳ from test/tests/checkpoint merging variable.ans:1:1 in block: :l = 1…
 --# post run check #--
 --- text ---
 | {}"3: " {}"3" {}"" |

test/results/closure define nested.ans

@@ -5,6 +5,6 @@
 no variable "y" defined in closure
 	↳ from test/tests/closure define nested.ans:12:4 in call: f . "y"
 	↳ from test/tests/closure define nested.ans:12:1 in text interpolation: | {f . "y"} |
-	↳ from ? in block: :f = ($() _)…
+	↳ from test/tests/closure define nested.ans:1:1 in block: :f = ($() _)…
 --# saved #--
 {}

test/results/constant variable.ans

@@ -2,6 +2,6 @@
 --- error ---
 trying to change the value of constant a
 	↳ from test/tests/constant variable.ans:5:3 in call: a = 52
-	↳ from ? in block: ::a = 3…
+	↳ from test/tests/constant variable.ans:1:1 in block: ::a = 3…
 --# saved #--
 {}

test/results/constrained variable assignement.ans

@@ -6,6 +6,6 @@
 --- error ---
 value check failure for weigh; 32 does not satisfy $(x) type(x) == t
 	↳ from test/tests/constrained variable assignement.ans:9:7 in call: weigh = 32
-	↳ from ? in block: :weigh::is("kg") = type(5, "kg")…
+	↳ from test/tests/constrained variable assignement.ans:1:1 in block: :weigh::is("kg") = type(5, "kg")…
 --# saved #--
 {}

test/results/define override function.ans

@@ -2,6 +2,6 @@
 --- error ---
 a is already defined in the current scope
 	↳ from test/tests/define override function.ans:4:4 in call: :a = 2
-	↳ from ? in block: :a = ($() _)…
+	↳ from test/tests/define override function.ans:1:1 in block: :a = ($() _)…
 --# saved #--
 {}

test/results/define override variable.ans

@@ -2,6 +2,6 @@
 --- error ---
 can't add an overload variant to non-overloadable variable a defined in the same scope
 	↳ from test/tests/define override variable.ans:3:1 in call: :a = ($() _)
-	↳ from ? in block: :a = 2…
+	↳ from test/tests/define override variable.ans:1:1 in block: :a = 2…
 --# saved #--
 {}

test/results/define override.ans

@@ -2,6 +2,6 @@
 --- error ---
 a is already defined in the current scope
 	↳ from test/tests/define override.ans:3:4 in call: :a = 2
-	↳ from ? in block: :a = 5…
+	↳ from test/tests/define override.ans:1:1 in block: :a = 5…
 --# saved #--
 {}

test/results/error.ans

@@ -2,6 +2,6 @@
 --- error ---
 nope
 	↳ from test/tests/error.ans:1:6 in call: error("nope")
-	↳ from ? in block: error("nope")
+	↳ from test/tests/error.ans:1:1 in block: error("nope")
 --# saved #--
 {}

test/results/exported variable nested.ans

@@ -8,6 +8,6 @@
 identifier "z" is undefined in branch test/tests/exported variable nested.ans - run
 	↳ from test/tests/exported variable nested.ans:12:3 in identifier: z
 	↳ from test/tests/exported variable nested.ans:12:1 in text interpolation: | {z} |
-	↳ from ? in block: :f = ($() _)…
+	↳ from test/tests/exported variable nested.ans:1:1 in block: :f = ($() _)…
 --# saved #--
 {}

test/results/for invalid iterator.ans

@@ -11,9 +11,9 @@
 		value check failure for parameter s
 	↳ from stdlib/for.ans:3:18 in call: iter(var)
 	↳ from stdlib/for.ans:3:12 in call: :iterator = iter(var)
-	↳ from stdlib/for.ans:2:1 in block: :iterator = iter(var)…
-	↳ from stdlib/for.ans:2:71 in call: _
+	↳ from stdlib/for.ans:3:1 in block: :iterator = iter(var)…
+	↳ from stdlib/for.ans:2:70 in call: _
 	↳ from test/tests/for invalid iterator.ans:1:4 in call: for(:x, 42)
-	↳ from ? in block: for(:x, 42)…
+	↳ from test/tests/for invalid iterator.ans:1:1 in block: for(:x, 42)…
 --# saved #--
 {}

test/results/function args arity check fail.ans

@@ -2,6 +2,6 @@
 --- error ---
 can't call function f: expected 2 arguments, received 1
 	↳ from test/tests/function args arity check fail.ans:4:2 in call: f("ok")
-	↳ from ? in block: :f = ($(a, b) _)…
+	↳ from test/tests/function args arity check fail.ans:1:1 in block: :f = ($(a, b) _)…
 --# saved #--
 {}

test/results/function conflict.ans

@@ -2,6 +2,6 @@
 --- error ---
 a function with parameters $(a, b) is already defined in the overload
 	↳ from test/tests/function conflict.ans:5:1 in call: :f = ($(a, b) 0)
-	↳ from ? in block: :f = ($(a, b) 0)…
+	↳ from test/tests/function conflict.ans:1:1 in block: :f = ($(a, b) 0)…
 --# saved #--
 {}

test/results/function custom type dispatch error.ans

@@ -10,6 +10,6 @@
 	• $(name::($(x) type(x) == t)) (from test/tests/function custom type dispatch error.ans:4:1):
 		value check failure for parameter name
 	↳ from test/tests/function custom type dispatch error.ans:14:2 in call: a(type(5, "nope"))
-	↳ from ? in block: :french name = "french name"…
+	↳ from test/tests/function custom type dispatch error.ans:1:1 in block: :french name = "french name"…
 --# saved #--
 {}

test/results/function scope wrong.ans

@@ -3,6 +3,6 @@
 identifier "b" is undefined in branch test/tests/function scope wrong.ans - run
 	↳ from test/tests/function scope wrong.ans:4:7 in identifier: b
 	↳ from test/tests/function scope wrong.ans:4:1 in text interpolation: | a: {b} |
-	↳ from ? in block: :a = ($() _)…
+	↳ from test/tests/function scope wrong.ans:1:1 in block: :a = ($() _)…
 --# saved #--
 {}

test/results/function separate variable from variants.ans

@@ -21,6 +21,6 @@
 		value check failure for parameter c
 	↳ from test/tests/function separate variable from variants.ans:10:4 in call: f . "a"
 	↳ from test/tests/function separate variable from variants.ans:10:1 in text interpolation: | {f . "a"} = 2 |
-	↳ from ? in block: :f = ($() _)…
+	↳ from test/tests/function separate variable from variants.ans:1:1 in block: :f = ($() _)…
 --# saved #--
 {}

test/results/function type dispatch ambigous.ans

@@ -4,6 +4,6 @@
 	• $(x::is number) (from test/tests/function type dispatch ambigous.ans:1:1)
 	• $(a::is number) (from test/tests/function type dispatch ambigous.ans:4:1)
 	↳ from test/tests/function type dispatch ambigous.ans:7:3 in call: fn(5)
-	↳ from ? in block: :fn = ($(x::is number) _)…
+	↳ from test/tests/function type dispatch ambigous.ans:1:1 in block: :fn = ($(x::is number) _)…
 --# saved #--
 {}

test/results/list assignement.ans

@@ -12,6 +12,6 @@
 --- error ---
 list index out of bounds
 	↳ from test/tests/list assignement.ans:21:6 in call: x(5) = 0
-	↳ from ? in block: :x = *[1, 2]…
+	↳ from test/tests/list assignement.ans:1:1 in block: :x = *[1, 2]…
 --# saved #--
 {}

test/results/list index.ans

@@ -11,6 +11,6 @@
 tuple index out of bounds
 	↳ from test/tests/list index.ans:11:4 in call: x(-4)
 	↳ from test/tests/list index.ans:11:1 in text interpolation: | {x(-4)} |
-	↳ from ? in block: :x = [1, 2, 3]…
+	↳ from test/tests/list index.ans:1:1 in block: :x = [1, 2, 3]…
 --# saved #--
 {}

test/results/load file error.ans

@@ -3,6 +3,6 @@
 no variable "b" defined in environment
 	↳ from test/tests/load file error.ans:3:4 in call: e . "b"
 	↳ from test/tests/load file error.ans:3:1 in text interpolation: | {e . "b"} |
-	↳ from ? in block: :e = load("test/tests/import/test.ans")…
+	↳ from test/tests/load file error.ans:1:1 in block: :e = load("test/tests/import/test.ans")…
 --# saved #--
 {}

test/results/merge nested mutable error bis.ans

@@ -2,18 +2,18 @@
 --- error ---
 abort
 	↳ from test/tests/merge nested mutable error bis.ans:14:7 in call: error("abort")
-	↳ from test/tests/merge nested mutable error bis.ans:3:1 in block: insert(a, b)…
+	↳ from test/tests/merge nested mutable error bis.ans:4:1 in block: insert(a, b)…
 	↳ from test/tests/merge nested mutable error bis.ans:3:18 in call: _
 	↳ from stdlib/script.ans:31:6 in call: fn!
-	↳ from stdlib/script.ans:29:3 in block: resume target = ()…
+	↳ from stdlib/script.ans:30:1 in block: resume target = ()…
 	↳ from stdlib/script.ans:29:7 in call: else!
-	↳ from stdlib/script.ans:25:2 in block: if(fn . "current checkpoint")…
-	↳ from stdlib/script.ans:25:9 in call: _
+	↳ from stdlib/script.ans:26:1 in block: if(fn . "current checkpoint")…
+	↳ from stdlib/script.ans:25:8 in call: _
 	↳ from stdlib/script.ans:39:9 in call: value(s)!
-	↳ from stdlib/script.ans:38:1 in block: value(s)!
-	↳ from stdlib/script.ans:38:20 in call: _
+	↳ from stdlib/script.ans:39:1 in block: value(s)!
+	↳ from stdlib/script.ans:38:19 in call: _
 	↳ from test/tests/merge nested mutable error bis.ans:19:2 in call: f!
-	↳ from ? in block: :a = *[1]…
+	↳ from test/tests/merge nested mutable error bis.ans:1:1 in block: :a = *[1]…
 --# post run check #--
 --- text ---
 | {}"[1,[2,3]]: " {}"*[1, *[2, 3]]" {}"" |

test/results/merge nested mutable error.ans

@@ -2,18 +2,18 @@
 --- error ---
 abort
 	↳ from test/tests/merge nested mutable error.ans:14:7 in call: error("abort")
-	↳ from test/tests/merge nested mutable error.ans:3:1 in block: insert(a, b)…
+	↳ from test/tests/merge nested mutable error.ans:4:1 in block: insert(a, b)…
 	↳ from test/tests/merge nested mutable error.ans:3:18 in call: _
 	↳ from stdlib/script.ans:31:6 in call: fn!
-	↳ from stdlib/script.ans:29:3 in block: resume target = ()…
+	↳ from stdlib/script.ans:30:1 in block: resume target = ()…
 	↳ from stdlib/script.ans:29:7 in call: else!
-	↳ from stdlib/script.ans:25:2 in block: if(fn . "current checkpoint")…
-	↳ from stdlib/script.ans:25:9 in call: _
+	↳ from stdlib/script.ans:26:1 in block: if(fn . "current checkpoint")…
+	↳ from stdlib/script.ans:25:8 in call: _
 	↳ from stdlib/script.ans:39:9 in call: value(s)!
-	↳ from stdlib/script.ans:38:1 in block: value(s)!
-	↳ from stdlib/script.ans:38:20 in call: _
+	↳ from stdlib/script.ans:39:1 in block: value(s)!
+	↳ from stdlib/script.ans:38:19 in call: _
 	↳ from test/tests/merge nested mutable error.ans:19:2 in call: f!
-	↳ from ? in block: :a = *[1]…
+	↳ from test/tests/merge nested mutable error.ans:1:1 in block: :a = *[1]…
 --# post run check #--
 --- text ---
 | {}"[1,[2,3]]: " {}"*[1, *[2, 3]]" {}"" |

test/results/scope checkpoint mutable bis error.ans

@@ -16,15 +16,15 @@
 --- error ---
 t
 	↳ from test/tests/scope checkpoint mutable bis error.ans:32:7 in call: error("t")
-	↳ from test/tests/scope checkpoint mutable bis error.ans:7:1 in block: insert(t, len(l) + 1)…
-	↳ from test/tests/scope checkpoint mutable bis error.ans:7:8 in call: _
+	↳ from test/tests/scope checkpoint mutable bis error.ans:8:1 in block: insert(t, len(l) + 1)…
+	↳ from test/tests/scope checkpoint mutable bis error.ans:7:7 in call: _
 	↳ from test/tests/scope checkpoint mutable bis error.ans:19:4 in call: f(t)
-	↳ from test/tests/scope checkpoint mutable bis error.ans:15:2 in block: | REC |…
+	↳ from test/tests/scope checkpoint mutable bis error.ans:16:1 in block: | REC |…
 	↳ from test/tests/scope checkpoint mutable bis error.ans:15:4 in call: if(n < 1)
-	↳ from test/tests/scope checkpoint mutable bis error.ans:7:1 in block: insert(t, len(l) + 1)…
-	↳ from test/tests/scope checkpoint mutable bis error.ans:7:8 in call: _
+	↳ from test/tests/scope checkpoint mutable bis error.ans:8:1 in block: insert(t, len(l) + 1)…
+	↳ from test/tests/scope checkpoint mutable bis error.ans:7:7 in call: _
 	↳ from test/tests/scope checkpoint mutable bis error.ans:41:2 in call: f(l)
-	↳ from ? in block: :x = *[99]…
+	↳ from test/tests/scope checkpoint mutable bis error.ans:1:1 in block: :x = *[99]…
 --# post run check #--
 --- text ---
 | {}"AFTER ERROR" |

test/results/scope checkpoint mutable error.ans

@@ -12,15 +12,15 @@
 --- error ---
 t
 	↳ from test/tests/scope checkpoint mutable error.ans:23:7 in call: error("t")
-	↳ from test/tests/scope checkpoint mutable error.ans:5:1 in block: insert(t, len(l) + 1)…
-	↳ from test/tests/scope checkpoint mutable error.ans:5:8 in call: _
+	↳ from test/tests/scope checkpoint mutable error.ans:6:1 in block: insert(t, len(l) + 1)…
+	↳ from test/tests/scope checkpoint mutable error.ans:5:7 in call: _
 	↳ from test/tests/scope checkpoint mutable error.ans:17:4 in call: f(t)
-	↳ from test/tests/scope checkpoint mutable error.ans:13:2 in block: | REC |…
+	↳ from test/tests/scope checkpoint mutable error.ans:14:1 in block: | REC |…
 	↳ from test/tests/scope checkpoint mutable error.ans:13:4 in call: if(n < 1)
-	↳ from test/tests/scope checkpoint mutable error.ans:5:1 in block: insert(t, len(l) + 1)…
-	↳ from test/tests/scope checkpoint mutable error.ans:5:8 in call: _
+	↳ from test/tests/scope checkpoint mutable error.ans:6:1 in block: insert(t, len(l) + 1)…
+	↳ from test/tests/scope checkpoint mutable error.ans:5:7 in call: _
 	↳ from test/tests/scope checkpoint mutable error.ans:32:2 in call: f(l)
-	↳ from ? in block: :l = *[1]…
+	↳ from test/tests/scope checkpoint mutable error.ans:1:1 in block: :l = *[1]…
 --# post run check #--
 --- text ---
 | {}"AFTER ERROR" |

test/results/scope checkpoint mutable ter error.ans

@@ -16,15 +16,15 @@
 --- error ---
 t
 	↳ from test/tests/scope checkpoint mutable ter error.ans:34:7 in call: error("t")
-	↳ from test/tests/scope checkpoint mutable ter error.ans:7:1 in block: insert(t, len(l) + 1)…
-	↳ from test/tests/scope checkpoint mutable ter error.ans:7:8 in call: _
+	↳ from test/tests/scope checkpoint mutable ter error.ans:8:1 in block: insert(t, len(l) + 1)…
+	↳ from test/tests/scope checkpoint mutable ter error.ans:7:7 in call: _
 	↳ from test/tests/scope checkpoint mutable ter error.ans:19:4 in call: f(t)
-	↳ from test/tests/scope checkpoint mutable ter error.ans:15:2 in block: | REC |…
+	↳ from test/tests/scope checkpoint mutable ter error.ans:16:1 in block: | REC |…
 	↳ from test/tests/scope checkpoint mutable ter error.ans:15:4 in call: if(n < 1)
-	↳ from test/tests/scope checkpoint mutable ter error.ans:7:1 in block: insert(t, len(l) + 1)…
-	↳ from test/tests/scope checkpoint mutable ter error.ans:7:8 in call: _
+	↳ from test/tests/scope checkpoint mutable ter error.ans:8:1 in block: insert(t, len(l) + 1)…
+	↳ from test/tests/scope checkpoint mutable ter error.ans:7:7 in call: _
 	↳ from test/tests/scope checkpoint mutable ter error.ans:43:2 in call: f(l)
-	↳ from ? in block: :x = *[99]…
+	↳ from test/tests/scope checkpoint mutable ter error.ans:1:1 in block: :x = *[99]…
 --# post run check #--
 --- text ---
 | {}"AFTER ERROR" |

test/results/symbol alias constant.ans

@@ -7,6 +7,6 @@
 --- error ---
 trying to change the value of constant d
 	↳ from test/tests/symbol alias constant.ans:12:3 in call: d = 5
-	↳ from ? in block: :l = *[1, 2, 3]…
+	↳ from test/tests/symbol alias constant.ans:1:1 in block: :l = *[1, 2, 3]…
 --# saved #--
 {}