still fighting async recursion...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>

v3/package-lock.json

@@ -32,9 +32,9 @@
       }
     },
     "node_modules/brace-expansion": {
-      "version": "5.0.6",
+      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
-      "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==",
+      "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^4.0.2"
@@ -231,9 +231,9 @@
       }
     },
     "node_modules/ig-types": {
-      "version": "6.26.2",
+      "version": "6.26.3",
-      "resolved": "https://registry.npmjs.org/ig-types/-/ig-types-6.26.2.tgz",
+      "resolved": "https://registry.npmjs.org/ig-types/-/ig-types-6.26.3.tgz",
-      "integrity": "sha512-VGg8MVmpblVCmZK52bJcDtPG3uRFiEyPnlDEIJv3MymTm+aUNWR/Th20NU2wfGV/Ux3KlsbGlhe+0c9ZKTYPDw==",
+      "integrity": "sha512-WDGHnC2up/7x5agsS2HfQbegfY97mMGM4xNv1HII3OiidcoPFDjp7nLdOOsWfZOtXA7nGWdAriGeVCOOJxEhew==",
       "license": "BSD-3-Clause",
       "dependencies": {
         "ig-object": "^6.0.0",

v3/pwiki/parser.js

@@ -608,6 +608,9 @@ module.BaseParser = {
 	// 			sync (text) and async (DOM) rendering
 	// 			in the simplest form: take the expanded AST and merge 
 	// 			into a single string
+	// XXX BUG: .wait can resolve before everything in the tree is resolved...
+	// 		to reproduce:
+	// 			@include(/async/recursive/SelfOther recursive="recursion found")
 	expand: function(page, ast, state={}, nested_handlers={}){
 		var that = this
 		ast = typeof(ast) != 'object' ?
@@ -1263,6 +1266,8 @@ module.parser = {
 		// 		to the same context...
 		// XXX need a way to make encode option transparent...
 		// XXX do we want to load a specific slot/block???
+		// XXX REVISE...
+		INCLUDE_LIMIT: 20,
 		include: Macro(
 			['src', 'recursive', 'join', 
 				['s', 'strict', 'isolated']],
@@ -1279,13 +1284,6 @@ module.parser = {
 			function(page, args, body, state, handler){
 				var that = this
 
-				/* XXX see .joinBlocks(..) join caching for more info...
-				// cache body ast...
-				body = body ?
-					this.ast(body)
-					: body
-				//*/
-
 				var recursive = 
 				state.recursive = 
 					args.recursive 
@@ -1300,19 +1298,6 @@ module.parser = {
 				return Promise.awaitOrRun(
 					this.parseNested(page, src, state),
 					function(src){
-						// check for recursion...
-						// XXX will this work for @source(..)???
-						var stack = state.include_stack ??= []
-						if(stack.includes(src)){
-							if(!recursive){
-								throw new Error('Recursive macro: '+stack) }
-							return that.expand(page, recursive, state) }
-						stack.push(src)
-
-						var cache = state.cache ??= {}
-						if(cache[src]){
-							return cache[src] }
-
 						// XXX should this be a tree??
 						// 		...need to at least split direct and 
 						// 		indirect dependencies...
@@ -1325,60 +1310,75 @@ module.parser = {
 
 						// content handler...
 						handler ??= 
-							function(page, src, text, state){
+							function(page, body, path, text, state){
-								page = page.get(src)
+								// re-include limit...
-								return args.isolated ?
+								// XXX HACK???
+								if( ++(state.included ??= {[path]: 0})[path] 
+										> this.macros.INCLUDE_LIMIT ?? 20){
+									// XXX BUG: for some reason for async recursion this
+									// 		breaks returning [object Object] overriding
+									// 		the actual return value
+									if(!recursive){
+										throw new Error(path +': include limit reached: '+ state.included[path]) }
+									return that.expand(page, recursive, state) }
+								// handle recursion...
+								// XXX BUG: for some reason this does not work for async...
+								// 		...and works quite differently in tests and 
+								// 		in console -- returns [object Object] in the
+								// 		former and hangs in the later...
+ 								var include_stack = state.include_stack ??= []
+								if(include_stack.includes(path)){
+									if(!recursive){
+										throw new Error('Recursive macro: '+include_stack) }
+									return that.expand(page, recursive, state) }
+								include_stack.push(path)
+
+								// XXX check cache???
+
+								var res = args.isolated ?
 									this.resolve(
 										page, 
 										text, 
-										Object.assign(
+										args.isolated == 'partial' ?
-											args.isolated == 'partial' ?
+											serialize.partialDeepCopy(state)
-												serialize.partialDeepCopy(state)
+											: {})
-												: {},
+									: this.expand(page, text, state)
-											{stack: state.stack ?? []}))
+
-									: this.expand(page, text, state) }
+								Promise.awaitOrRun(
+									state.waitAll,
+									function(){
+										// handle recursion...
+										state.include_stack.at(-1) == src
+											&& state.include_stack.pop()
+										// cleanup...
+										if(state.include_stack.length == 0){
+											delete state.include_stack
+											delete state.recursive } }) 
+
+								return res }
 
 						var pageHandler =
 							function([path, text]){
-								var content_handled = false
+								// handle nested promises...
-								return body ?
+								return Promise.awaitOrRun(
-									// handle body / <content/>...
+									text,
-									Promise.awaitOrRun(
+									function(text){
-										that.expand(page.get(path), body, state, 
+										return handler.call(that, page, body, path, text, state) }) }
-											{ content: function(){
-												content_handled = true
-												return text = handler.call(that, page, src, text, state) } }),
-										function(text){
-											// if no <content/> present we still 
-											// need to handle the included page...
-											content_handled 
-												|| handler.call(that, page, src, text, state)
-											return text })
-									// place as-is...
-									: handler.call(that, page, src, text, state) }
-						var resultHandler =
-							function(pages){
-								state.include_stack.at(-1) == src
-									&& state.include_stack.pop()
-								// cleanup...
-								if(state.include_stack.length == 0){
-									delete state.include_stack
-									delete state.recursive }
-								// cache the final result...
-								cache[src] = pages
-						   		return pages }
 
 						// get and run things...
 						return Promise.awaitOrRun(
 							page.get(src).matched,
 							page.get(src).raw,
-							function(paths, pages){
+							function(paths, texts){
-								paths = paths instanceof Array ?
+								texts = 
-									paths
+									// XXX how do we handle paths returning non-strings???
-									: [paths]
+									// special case: list page...
-								pages = pages instanceof Array ?
+									paths.length == 1
-									pages
+											&& texts instanceof Array ?
-									: [pages]
+										[texts]
+									: texts instanceof Array ?
+										texts
+									: [texts ?? '']
 								return Promise.awaitOrRun(
 									// handle pages...
 									Promise
@@ -1386,13 +1386,17 @@ module.parser = {
 											that.joinBlocks(
 												page,
 												Array
-													.zip(paths, pages)
+													.zip(paths, texts)
 													.map(pageHandler),
 												args.join,
 												state))
 										.flat()
 										.sync(),
-									resultHandler ) }) }) })),
+									function(pages){
+										/* XXX cache the final result...
+										cache[src] = pages
+										//*/
+										return pages }) }) }) })),
 
 		// NOTE: the main difference between this and @include is that 
 		// 		this renders the src in the context of current page while 
@@ -1544,6 +1548,7 @@ module.parser = {
 		// 			...this seems to effect non-pattern pages...
 		// XXX should macro:index be 0 or 1 (current) based???
 		// XXX SORT sorting not implemented yet...
+		// XXX check macro recursion...
 		macro: Macro(
 			['name', 'src', 'sort', 'text', 'join', 'else',
 				['strict', 'isolated', 'inheritmacros', 'inheritvars']],
@@ -1561,7 +1566,38 @@ module.parser = {
 							body = (state.macros ?? {})[name] }
 						return args.src && body ?
 							// run macro...
-							that.macros.include.call(that, page, args, body, state)
+							that.macros.include.call(that, page, args, body, state,
+								function(page, body, path, text, state){
+									var that = this
+
+									var handle = function(page, text, state){
+										return args.isolated ?
+											that.resolve(
+												page, 
+												text, 
+												Object.assign(
+													args.isolated == 'partial' ?
+														serialize.partialDeepCopy(state)
+														: {},
+													{include_stack: state.include_stack ?? []}))
+											: that.expand(page, text, state) }
+
+									//var content_handled = false
+									return body ?
+										// handle body / <content/>...
+										Promise.awaitOrRun(
+											that.expand(page.get(path), body, state, 
+												{ content: function(){
+													content_handled = true
+													return text = handle(page, text, state) } }),
+											function(text){
+												// if no <content/> present we still 
+												// need to handle the included page...
+												//content_handled 
+												//	|| handle.call(that, page, text, state)
+												return text })
+										// place as-is...
+										: handle(page, text, state) })
 							: '' }) })),
 
 		// nesting rules...

v3/pwiki/test/parser.js

@@ -87,7 +87,13 @@ module.exports.P = {
 
 		return res.length == 1 ?
 			res[0]
+			/*/ if at least one page is a promise wait for all...
+			: Promise
+				.iter(res)
+				.sync() },
+			/*/
 			: res },
+			//*/
 
 	// XXX should this return an arrya for a multi-match???
 	get text(){
@@ -278,13 +284,14 @@ test.Setups({
 			],
 		} },
 	//*/
-	// content...
+	/*/ XXX content...
 	include_content: function(assert, path='/page'){
 		return {
 			page: P,
 			code: [
 				'<include "'+ path +'">[[ <content/> ]]</include>',
 					'[[ '+ P.get(path).raw +' ]]', ], } },
+	//*/
 	// XXX
 	
 	// recursion...
@@ -294,6 +301,7 @@ test.Setups({
 			page: P,
 			code:[
 				'@include(/recursive/Self recursive="recursion found")',
+				// XXX this hangs...
 				'@include(/async/recursive/Self recursive="recursion found")',
 					'<< recursion found >>', ], } },
 	include_recursive_b: function(assert){
@@ -301,6 +309,7 @@ test.Setups({
 			page: P,
 			code:[
 				'@include(/recursive/SelfOther recursive="recursion found")',
+				// XXX this hangs...
 				'@include(/async/recursive/SelfOther recursive="recursion found")',
 					'<< << recursion found >> >>', ], } },