refactoring + cleanup...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>

v3/pwiki/parser.js

@@ -247,6 +247,36 @@ module.BaseParser = {
 					?? args.text }
 		return macro.call(this, page, args, body, state, ...rest) },
 
+	// place join block between block elements...
+	joinBlocks: function(page, blocks, join, state){
+		var that = this
+		if(typeof(blocks) == 'string' 
+				|| join == null){
+			return blocks }
+		// we do not need to rebuild the ast for each use...
+		// XXX this can break things -- need to store the parse stage 
+		// 		in the structure so as to determine where we left off...
+		// 		...the problem is that we can't tell the difference 
+		// 		between a parsed and expanded stages -- one way, re-running 
+		// 		a stage is not an issue but the other way, skipping can 
+		// 		break...
+		//join = this.ast(page, join, state)
+		return blocks
+			.map(function(block, i, l){
+				return [
+					block,
+					...(i < l.length-1 ?
+						that.expand(page, join, state)
+						: []),
+				] })
+			.flat() },
+	filterBlocks: function(page, blocks, filter, state){
+		if(filter == null){
+			return blocks }
+
+		// XXX
+
+		return blocks },
 
 	// Strip comments...
 	//
@@ -283,7 +313,7 @@ module.BaseParser = {
 	//
 	// NOTE: this internally uses .macros' keys to generate the 
 	// 		lexing pattern.
-	lex: function*(page, str){
+	lex: function*(str){
 		str = typeof(str) != 'string' ?
 			str+''
 			: str
@@ -412,9 +442,9 @@ module.BaseParser = {
 	//
 	// NOTE: this internaly uses .macros to check for propper nesting
 	//group: function*(page, lex, to=false){
-	group: function*(page, lex, to=false, parent, context){
+	group: function*(lex, to=false, parent, context){
 		lex = typeof(lex) != 'object' ?
-			this.lex(page, lex)
+			this.lex(lex)
 			: lex
 
 		var quoting = to 
@@ -466,8 +496,8 @@ module.BaseParser = {
 
 			// open block...
 			if(value.type == 'opening'){
-				//value.body = [...this.group(page, lex, value.name)]
+				//value.body = [...this.group(lex, value.name)]
-				value.body = [...this.group(page, lex, value.name, value)]
+				value.body = [...this.group(lex, value.name, value)]
 				value.type = 'block'
 
 				// unify .body, .args.body and .args.text into .body...
@@ -477,7 +507,6 @@ module.BaseParser = {
 							?? value.args.text)){
 					value.body = 
 						[...this.group(
-							page,
 							value.args.body
 								?? value.args.text,
 							false,
@@ -571,7 +600,7 @@ module.BaseParser = {
 	expand: function(page, ast, state={}){
 		var that = this
 		ast = typeof(ast) != 'object' ?
-				this.group(page, ast)
+				this.group(ast)
 			: ast instanceof types.Generator ?
 				ast
 			: ast.iter()
@@ -607,6 +636,7 @@ module.BaseParser = {
 				continue }
 
 			// expand down...
+			// XXX cache this as an AST
 			body 
 				&& !that.macros[name].lazy
 				&& (body = this.expand(page, body, state))
@@ -680,7 +710,8 @@ module.BaseParser = {
 
 	// resolve stage II macros and merge results...
 	//
-	/* XXX 
+	// XXX DNF
+	//
 	// XXX these yeild different results:
 	// 			r = parser.resolve(
 	// 				tests.P, 
@@ -700,84 +731,6 @@ module.BaseParser = {
 	// 		XXX it looks like the first slot is resolved before the last 
 	// 			slot has a chance to set the value as it is waiting for 
 	// 			the first slot to finish...
-	resolve: function(page, ast, state={}){
-		var that = this
-		ast = typeof(ast) != 'object' ?
-				this.expand(page, ast, state)
-			: ast instanceof types.Generator ?
-				ast
-			: ast.iter()
-
-		// merge resolved elements into the last item of elems...
-		var  elems = []
-		var merge = function(...args){
-			var prev = ''
-			while(args.length > 0){
-				while(args.length > 0 
-						&& typeof(args[0]) != 'object'
-						&& typeof(args[0]) != 'function'){
-					prev += args.shift() }
-				// merged section...
-				if(prev.length > 0){
-					if(elems.length > 0 
-							&& typeof(elems.at(-1)) == 'string'){
-						elems[elems.length-1] += prev
-					} else {
-						elems.push(prev) }
-					prev = '' }
-				if(args.length > 0){
-					elems.push(args.shift()) } } }
-
-		// XXX can we make this more granular and wait only where we need 
-		// 		to wait?
-		// 		....the wait is logical as we need to return a string here 
-		// 		eventually.
-		// 		should this be split into .resolve(..) nad .merge(..)???
-		return Promise.awaitOrRun(
-			state.wait,
-			function(){
-				// XXX can ast be a promise???
-				// XXX elems can be unresolved -- need a merge strategy for them...
-				//var waiting = []
-				for(var elem of ast){
-					// nesting...
-					while(elem && elem.value){
-						elem = elem.value 
-						// exec stage II macros...
-						// XXX this should be called when all the promises before 
-						// 		are resolved...
-						if(typeof(elem) == 'function'){
-							elem = elem(state) } }
-					if(elem == null){
-						continue }
-					// atomic values...
-					if(typeof(elem) != 'object'){
-						merge(elem) 
-						continue }
-					// value is resolved but "empty" -> skip...
-					if('value' in elem 
-							&& (elem.value == null 
-								|| elem.value == '')){
-						continue }
-					// expand ast...
-					if(elem instanceof Array){
-						// XXX this can be or containe promises...
-						merge(...that.resolve(page, elem, state)) 
-						continue }
-					// expand .body attribute...
-					// XXX is this needed here???
-					if(elem.body instanceof Array){
-						console.warn('!!! RESOLVE_BODY')
-						// XXX this can be or containe promises...
-						elem.body = that.resolve(page, elem.body, state) }
-					// nested macro with no value set -- skip...
-					if(that.macros[elem.name] instanceof Array){
-						continue }
-					// unresolved...
-					merge(elem) }
-
-				return elems }) },
-	//*/
 	resolve: function(page, ast, state={}){
 		var that = this
 		ast = typeof(ast) != 'object' ?
@@ -1128,6 +1081,10 @@ function(macro){
 	return macro }
 
 
+
+
+
+
 // XXX RENAME...
 // 		...this is more of an expander/executer...
 // 		...might be a good idea to also do a check without executing...
@@ -1508,6 +1465,8 @@ module.parser = {
 		// 		not 100% correct manner focusing on path depth and ignoring
 		// 		the context, this potentially can lead to false positives.
 		//
+		// XXX might be a good idea to add a <content/> tag to place the 
+		// 		loaded text...
 		// XXX add path recursion test to data -- fail if two paths resolve 
 		// 		to the same context...
 		// XXX need a way to make encode option transparent...
@@ -1533,10 +1492,16 @@ module.parser = {
 			function(page, args, body, state, handler){
 				var that = this
 
+				/* XXX see .joinBlocks(..) join caching for more info...
+				// cache body ast...
+				body = body ?
+					this.ast(body)
+					: body
+				//*/
+
 				var recursive = 
 				state.recursive = 
 					args.recursive 
-						?? body
 						?? state.recursive
 
 				var base = page.basepath
@@ -1546,6 +1511,20 @@ module.parser = {
 				return Promise.awaitOrRun(
 					this.parseNested(page, src, state),
 					function(src){
+						// check for recursion...
+						// XXX do we do this for pattern paths???
+						// XXX this does not catch the /A/A/A/... recursion...
+						var stack = state.include_stack ??= []
+						// XXX is this the right separator???
+						// 		...need something that can't be in a path...
+						var base_src = base +'|'+ src
+						if(stack.includes(base_src)){
+							if(recursive){
+								return that.expand(page, recursive, state) }
+							throw new Error('Recursion:\n\t'+ 
+								[...stack, base_src].join('\n\t\t-> ')) }
+						stack.push(base_src)
+
 						var cache = state.cache ??= {}
 						if(cache[src]){
 							return cache[src] }
@@ -1560,20 +1539,6 @@ module.parser = {
 						// 		out of context...
 						var depends = ((state.depends ??= {})[src] ??= {})
 
-						// check for recursion...
-						// XXX do we do this for pattern paths???
-						// XXX this does not catch the /A/A/A/... recursion...
-						var stack = state.include_stack ??= []
-						// XXX is this the right separator???
-						// 		...need something that can't be in a path...
-						var base_src = base +'|'+ src
-						if(stack.includes(base_src)){
-							if(recursive){
-								return recursive }
-							throw new Error('Recursion:\n\t'+ 
-								[...stack, base_src].join('\n\t\t-> ')) }
-						stack.push(base_src)
-
 						// content handler...
 						handler ??= 
 							function(page, text, state){
@@ -1591,21 +1556,14 @@ module.parser = {
 									: this.expand(page, text, state)}
 
 						var pageHandler =
-							function(text, i, l){
+							function(text){
-								return [
+								// XXX handle body / <content/>...
-									handler.call(that, page, text, state),
+								// XXX
-									// join...
+								return handler.call(that, page, text, state) }
-									(args.join 
-											&& i < l.length - 1) ?
-										that.expand(page, args.join, state)
-										: [],
-								].flat() }
 						var resultHandler =
 							function(pages){
-								// XXX not sure if this can happen or why...
+								state.include_stack.at(-1) == base_src
-								if(state.include_stack.at(-1) != base_src){
+									&& state.include_stack.pop()
-									throw new Error('Include stack error') }
-								state.include_stack.pop()
 								// cleanup...
 								if(state.include_stack.length == 0){
 									delete state.include_stack
@@ -1624,8 +1582,12 @@ module.parser = {
 								return Promise.awaitOrRun(
 									// handle pages...
 									Promise
-										.iter(pages
+										.iter(
-											.map( pageHandler ))
+											that.joinBlocks(
+												page,
+												pages.map(pageHandler), 
+												args.join, 
+												state))
 										.flat()
 										.sync(),
 									resultHandler ) }) }) })),
@@ -1637,9 +1599,7 @@ module.parser = {
 		// 		i.e. for @include(PATH) the paths within the included page 
 		// 		are resolved relative to PATH while for @source(PATH) 
 		// 		relative to the page containing the @source(..) statement...
-		// XXX UPDATE...
 		source: Macro(
-			// XXX should this have the same args as include???
 			['src', 'recursive', 'join', 
 				['s', 'strict']],
 			//['src'],
@@ -1677,7 +1637,6 @@ module.parser = {
 					function(src){
 						var text = 
 							src ?
-								// can be async...
 								page.get(src).raw
 							: body ?
 								body
@@ -1685,13 +1644,15 @@ module.parser = {
 						return Promise.awaitOrRun(
 							text,
 							function(text){
-								// filter...
+								return that.joinBlocks(
-								if(args.filter){
+									page, 
-									// XXX
+									that.filterBlocks(
-								}
+										page, 
-								// XXX join...
+										text, 
-								// XXX
+										args.filter, 
-								return text }) }) })),
+										state), 
+									args.join, 
+									state) }) }) })),
 
 		//
 		// 	@quote(<src>)

v3/pwiki/test/parser.js

@@ -17,7 +17,16 @@ module.exports.PAGES = {
 	'/page': 'Page',
 	'/async/page': Promise.resolve('Page'),
 	'/includePage': '@include(/page)',
-	'/isolated': '@slot(slot original)'
+	'/isolated': '@slot(slot original)',
+
+	'/recursive/Self': '<< @include(/recursive/Self) >>',
+	'/recursive/OtherSelf': '<< @include(/recursive/SelfOther) >>',
+	'/recursive/SelfOther': '<< @include(/recursive/OtherSelf) >>',
+	'/async/recursive/Self': Promise.resolve('<< @include(/async/recursive/Self) >>'),
+	'/async/recursive/OtherSelf': Promise.resolve('<< @include(/async/recursive/SelfOther) >>'),
+	'/async/recursive/SelfOther': Promise.resolve('<< @include(/async/recursive/OtherSelf) >>'),
+
+	'/multi/page': [ 'A', 'B', 'C' ],
 }
 
 var P = 
@@ -216,8 +225,26 @@ test.Setups({
 					'original overloaded',
 			],
 		} },
-	// XXX recursion...
+	// recursion...
-	// XXX
+	// XXX test path recursion: /A -> /A/A -> /A/A/A -> ...
+	include_recursive_a: function(assert){
+		return {
+			page: P,
+			code:[
+				'@include(/recursive/Self recursive="recursion found")',
+				'@include(/async/recursive/Self recursive="recursion found")',
+					'<< recursion found >>', ], } },
+	include_recursive_b: function(assert){
+		return {
+			page: P,
+			code:[
+				'@include(/recursive/SelfOther recursive="recursion found")',
+				'@include(/async/recursive/SelfOther recursive="recursion found")',
+					'<< << recursion found >> >>', ], } },
+	
+	// quote...
+	// for inline quoting see: test.Modifiers.quote
+	// XXX <quote src=.. />
 })
 
 
@@ -246,32 +273,49 @@ test.Modifiers({
 			`[[ ${state.code.at(-1)} ]]`,
 		]
 		return state },
+	quote: function(assert, state){
+		return state.code
+			.slice(0, -1)
+			.map(function(code){
+				code = code.replace(/<\/quote>/, '&lt;/quote&gt;')
+				return {
+					page: state.P,
+					code: [
+						`<quote>${ code }</quote>`,
+						code, 
+					], 
+				} }) },
 })
 
 
 test.Tests({
 	parse: async function(assert, state){
-		var {page, code, st} = state
+		var states = 
-		page ??= {}
+			state instanceof Array ?
-		st ??= {}
+				state
+				: [state]
+		for(state of states){
+			var {page, code, st} = state
+			page ??= {}
+			st ??= {}
 
-		var res
+			var res
-		var inputs = code.slice(0, -1)
+			var inputs = code.slice(0, -1)
-		var expect = code.at(-1)
+			var expect = code.at(-1)
-		var i = 0
+			var i = 0
-		for(var input of inputs){	
+			for(var input of inputs){	
-			var p = serialize.partialDeepCopy(page)
+				var p = serialize.partialDeepCopy(page)
-			var s = serialize.partialDeepCopy(st)
+				var s = serialize.partialDeepCopy(st)
-			assert(
+				assert(
-				(res = await parser.parse(
+					(res = await parser.parse(
-						p,
+							p,
-						input,
+							input,
-						s))
+							s))
-					=== expect,
+						=== expect,
-				'Parsing:',
+					'Parsing:',
-					'\n\t      in: "'+ input +'"',
+						'\n\t      in: "'+ input +'"',
-					'\n\t     out: "'+ res +'"',
+						'\n\t     out: "'+ res +'"',
-					'\n\texpected: "'+ expect +'"') } },
+						'\n\texpected: "'+ expect +'"') } } },
 	//asyncParse: async function(assert, state){
 	//	return await this.parse(assert, state) },
 })