started @quote(..), @source(..), @load(..) -- some of v2 seems to be a tad overcomplicated...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>

v3/pwiki/parser.js

@@ -418,7 +418,7 @@ module.BaseParser = {
 			: lex
 
 		var quoting = to 
-			&& (page.QUOTING_MACROS ?? []).includes(to)
+			&& !!this.macros[to].quoting
 			&& []
 
 		// NOTE: we are not using for .. of .. here as it depletes the 
@@ -530,6 +530,8 @@ module.BaseParser = {
 	//
 	//		// wait for all...
 	//		wait: <promise> | null,
+	//
+	//		...
 	//	}
 	//
 	//
@@ -574,14 +576,6 @@ module.BaseParser = {
 				ast
 			: ast.iter()
 
-		var wait = new Set()
-		state.wait instanceof Promise 
-			&& wait.add(state.wait)
-		var resolve, reject
-		state.wait = new Promise(
-			function(){
-				;[resolve, reject] = arguments })
-
 		var elems = []
 		for(let elem of ast){
 			// text block...
@@ -621,9 +615,10 @@ module.BaseParser = {
 			var res = that.callMacro(page, name, args, body, state)
 			// async...
 			if(res instanceof Promise){
+				let all, nested
+				all = state.waitAll = 
+					Promise.all([state.waitAll, res])
 				elem.resolving = res
-				wait.add(res)
-				state.waitAll = res
 				// XXX do we need to wait till the last .waitNested is 
 				// 		resolved?
 				// 		...should it's handlers complete??
@@ -632,13 +627,13 @@ module.BaseParser = {
 				// 		func to the macro...
 				// 		...do we need this?
 				if(!res.isolated){
-					state.waitNested = res }
+					nested = state.waitNested = 
+						Promise.all([state.waitNested, res]) }
 				res.then(
 					function(value){
-						wait.delete(res)
-						if(state.waitAll === res){
+						if(state.waitAll === all){
 							delete state.waitAll }
-						if(state.waitNested === res){
+						if(state.waitNested === nested){
 							delete state.waitNested }
 						delete elem.resolving
 						elem.value = value 
@@ -659,25 +654,52 @@ module.BaseParser = {
 				elem = res }
 			elems.push(elem) }
 
-		// handle wait for isolated macros to resolve...
-		var done = function(){
-			resolve(state)
-			delete state.wait }
-		if(wait.size > 0){
-			Promise.all(wait)
-				.then(
-					done,
-					function(err){
-						reject(err) }) 
-		// done...
-		} else {
-			done() }
+		// cleanup...
+		var wait
+		var waitAll = state.waitAll
+		state.waitAll
+			&& state.waitAll
+				.then(function(){
+					// only cleanup our own mess =)
+					waitAll === state.waitAll
+						&& (delete state.waitAll) })
+			&& (wait = state.wait = waitAll
+				.then(function(){ 
+					wait === state.wait
+						&& (delete state.wait)
+					return elems }))
+		var waitNested = state.waitNested
+		state.waitNested
+			&& state.waitNested
+				.then(function(){
+					// only cleanup our own mess =)
+					waitNested === state.waitNested
+						&& (delete state.waitNested) })
 
 		return elems },
 
 	// resolve stage II macros and merge results...
 	//
-	// XXX BUG: sync/async paths seem to diverge...
+	/* XXX 
+	// XXX these yeild different results:
+	// 			r = parser.resolve(
+	// 				tests.P, 
+	// 				'<slot X>@include(/async/page)</slot> <slot X value>',
+	// 				s = {})
+	// 			s.wait.then(function(){
+	// 				console.log(r) }) // -> ['Page '] (err)
+	// 		and:
+	// 			r = parser.expand(
+	// 				tests.P, 
+	// 				'<slot X>@include(/async/page)</slot> <slot X slot/>',
+	// 				s = {})
+	// 			s.wait.then(function(){
+	// 				r = parser.resolve(test.P, r, s)
+	// 				console.log(r) }) // -> ['slot '] (correct)
+	// 		-> promises seem to not be sequenced correctly here...
+	// 		XXX it looks like the first slot is resolved before the last 
+	// 			slot has a chance to set the value as it is waiting for 
+	// 			the first slot to finish...
 	resolve: function(page, ast, state={}){
 		var that = this
 		ast = typeof(ast) != 'object' ?
@@ -705,29 +727,89 @@ module.BaseParser = {
 					prev = '' }
 				if(args.length > 0){
 					elems.push(args.shift()) } } }
-		var unpack = function(elem){
-			// XXX
-		}
 
+		// XXX can we make this more granular and wait only where we need 
+		// 		to wait?
+		// 		....the wait is logical as we need to return a string here 
+		// 		eventually.
+		// 		should this be split into .resolve(..) nad .merge(..)???
+		return Promise.awaitOrRun(
+			state.wait,
+			function(){
+				// XXX can ast be a promise???
+				// XXX elems can be unresolved -- need a merge strategy for them...
+				//var waiting = []
+				for(var elem of ast){
+					// nesting...
+					while(elem && elem.value){
+						elem = elem.value 
+						// exec stage II macros...
+						// XXX this should be called when all the promises before 
+						// 		are resolved...
+						if(typeof(elem) == 'function'){
+							elem = elem(state) } }
+					if(elem == null){
+						continue }
+					// atomic values...
+					if(typeof(elem) != 'object'){
+						merge(elem) 
+						continue }
+					// value is resolved but "empty" -> skip...
+					if('value' in elem 
+							&& (elem.value == null 
+								|| elem.value == '')){
+						continue }
+					// expand ast...
+					if(elem instanceof Array){
+						// XXX this can be or containe promises...
+						merge(...that.resolve(page, elem, state)) 
+						continue }
+					// expand .body attribute...
+					// XXX is this needed here???
+					if(elem.body instanceof Array){
+						console.warn('!!! RESOLVE_BODY')
+						// XXX this can be or containe promises...
+						elem.body = that.resolve(page, elem.body, state) }
+					// nested macro with no value set -- skip...
+					if(that.macros[elem.name] instanceof Array){
+						continue }
+					// unresolved...
+					merge(elem) }
+
+				return elems }) },
+	//*/
+	resolve: function(page, ast, state={}){
+		var that = this
+		ast = typeof(ast) != 'object' ?
+				this.expand(page, ast, state)
+			: ast instanceof types.Generator ?
+				ast
+			: ast.iter()
+
+		// merge resolved elements into the last item of elems...
 		// XXX can ast be a promise???
 		// XXX elems can be unresolved -- need a merge strategy for them...
+		var  elems = []
 		for(var elem of ast){
 			// nesting...
 			while(elem && elem.value){
-				elem = elem.value 
 				// exec stage II macros...
-				if(typeof(elem) == 'function'){
-					elem = elem(state) } }
+				if(typeof(elem.value) == 'function'){
+					let e = elem
+					// NOTE: if not everything is resolved, delay the stage II
+					// 		callbacks till .wait is done...
+					Promise.awaitOrRun(
+						state.wait,
+						function(){
+							return elem = e.value = 
+								e.value(state) }) 
+					break } 
+				elem = elem.value }
 			if(elem == null){
 				continue }
-			// XXX do a delayed merge...
-			// 		...but for this we need to also apply the rest of this iteration to the result...
-			if(elem.resolving){
-				console.log('!!!!!!!!!!!!!!!!!!!')
-			}
 			// atomic values...
 			if(typeof(elem) != 'object'){
-				merge(elem) 
+				elems.push(elem) 
 				continue }
 			// value is resolved but "empty" -> skip...
 			if('value' in elem 
@@ -736,61 +818,48 @@ module.BaseParser = {
 				continue }
 			// expand ast...
 			if(elem instanceof Array){
-				// XXX this can containe promises...
-				merge(...this.resolve(page, elem, state)) 
+				// XXX this can be or containe promises...
+				elems.push(...that.resolve(page, elem, state)) 
 				continue }
 			// expand .body attribute...
-			if((elem.attrs ?? {}).body instanceof Array){
-				// XXX this can containe promises...
-				elem.attrs.body = this.resolve(page, elem.attrs.body, state) }
+			/* XXX is this needed here???
+			if(elem.body instanceof Array){
+				console.warn('!!! RESOLVE_BODY')
+				// XXX this can be or containe promises...
+				elem.body = that.resolve(page, elem.body, state) }
+			//*/
 			// nested macro with no value set -- skip...
-			if(this.macros[elem.name] instanceof Array){
+			if(that.macros[elem.name] instanceof Array){
 				continue }
 			// unresolved...
-			merge(elem) }
+			elems.push(elem) }
 
 		return elems },
 
+
 	// XXX render api...
 	// XXX how should this play with filters???
 	// 		...should filters be client-side only??
 	render: function*(page, ast, callback, state={}){
-		ast = this.resolve(page, ast, state)
-		for(var elem of ast){
-			yield Promise.awaitOrRun(
-				elem,
-				callback) } },
+		// XXX
+	},
 
 	// XXX
-	_parse: function(page, ast, state={}){
+	parse: function(page, ast, state={}, wait='wait'){
+		var that = this
+		var reresolve = !!state.wait
 		return Promise.awaitOrRun(
 			this.resolve(page, ast, state),
-			function(ast){
-				// XXX
-				if(ast.length > 1){
-					throw new Error('!!!!')
-				}
-				return ast[0] 
-					?? '' }) },
-	// XXX
+			state[wait],
+			function(ast, reresolve){
+				ast = reresolve ?
+					that.resolve(page, ast, state)
+					: ast
+				return (ast ?? '').join('') }) },
 	// XXX how should this play with filters???
 	// 		...should filters be client-side only??
 	parseNested: function(page, ast, state={}){
-		return Promise.awaitOrRun(
-			state.waitNested,
-			this._parse(page, ast, state),
-			function(_, ast){
-				return ast }) },
-
-	// XXX this can't be used from within macros -- will deadlock the results...
-	// XXX how should this play with filters???
-	// 		...should filters be client-side only??
-	parse: function(page, ast, state={}){
-		return Promise.awaitOrRun(
-			state.wait,
-			this._parse(page, ast, state),
-			function(_, ast){
-				return ast }) },
+		return this.parse(page, ast, state, 'waitNestedi') },
 
 
 
@@ -1052,6 +1121,12 @@ function(macro){
 	macro.lazy = true
 	return macro }
 
+var quoting =
+module.quoting =
+function(macro){
+	macro.quoting = true
+	return macro }
+
 
 // XXX RENAME...
 // 		...this is more of an expander/executer...
@@ -1060,9 +1135,6 @@ var parser =
 module.parser = {
 	__proto__: BaseParser,
 
-	// list of macros that will get raw text of their content...
-	QUOTING_MACROS: ['quote'],
-
 	//
 	// 	<macro>(<args>, <body>, <state>){ .. }
 	// 		-> undefined
@@ -1458,7 +1530,7 @@ module.parser = {
 			// 			is this a promise/value, iterable promise a generator
 			// 			an async generator, ... or a combination/stack of the above???
 			lazy(
-			function(page, args, body, state, key='included', handler){
+			function(page, args, body, state, handler){
 				var that = this
 
 				var recursive = 
@@ -1505,28 +1577,28 @@ module.parser = {
 						// content handler...
 						handler ??= 
 							function(page, text, state){
-								return this._parse(page, text, state) }
+								// XXX should this be the difference between 
+								// 		@include(..) and @source(..)???
+								// 		i.e. 
+								// 			.include(..) -> .resolve(..)
+								// 			.source(..) -> .expand(..)
+								return args.isolated ?
+									this.resolve(page, text, 
+										args.isolated == 'partial' ?
+											// XXX
+											serialize.partialDeepCopy(state)
+											: {}) 
+									: this.expand(page, text, state)}
 
 						var pageHandler =
 							function(text, i, l){
 								return [
-									handler.call(that, 
-										page, 
-										text, 
-										// isolated up -- will see all 
-										// the state but can have no 
-										// side-effects...
-										args.isolated == 'partial' ?
-												serialize.partialDeepCopy(state)
-											// fully isolated...
-											: args.isolated ?
-												{}
-											: state),
+									handler.call(that, page, text, state),
 									// join...
 									(args.join 
 											&& i < l.length - 1) ?
-										that._parse(page, args.join, state)
-										: []
+										that.expand(page, args.join, state)
+										: [],
 								].flat() }
 						var resultHandler =
 							function(pages){
@@ -1558,106 +1630,6 @@ module.parser = {
 										.sync(),
 									resultHandler ) }) }) })),
 
-		_include: Macro(
-			['src', 'recursive', 'join', 
-				['s', 'strict', 'isolated']],
-			async function*(parser, args, body, state, key='included', handler){
-				var macro = 'include'
-				if(typeof(args) == 'string'){
-					var [macro, args, body, state, key, handler] = arguments 
-					key = key ?? 'included' }
-				var base = this.get(this.path.split(/\*/).shift())
-				var src = args.src
-					&& this.resolvePathVars(
-						await base.parse(args.src, state))
-				if(!src){
-					return }
-				// XXX INHERIT_ARGS special-case: inherit args by default...
-				// XXX should this be done when isolated???
-				if(this.actions_inherit_args 
-						&& this.actions_inherit_args.has(pwpath.basename(src))
-						&& this.get(pwpath.dirname(src)).path == this.path){
-					src += ':$ARGS' }
-				var recursive = args.recursive ?? body
-				var isolated = args.isolated 
-				var strict = args.strict
-				var strquotes = args.s
-				var join = args.join 
-					&& await base.parse(args.join, state)
-
-				var depends = state.depends = 
-					state.depends 
-						?? new Set()
-				// XXX DEPENDS_PATTERN
-				depends.add(src)
-
-				handler = handler 
-					?? async function(src, state){
-						return isolated ?
-							//{data: await this.get(src)
-							{data: await this
-								.parse({
-									seen: state.seen, 
-									depends,
-									renderer: state.renderer,
-								})}
-							//: this.get(src)
-							: this
-								.parse(state) }
-
-				var first = true
-				for await (var page of this.get(src).asPages(strict)){
-					if(join && !first){
-						yield join }
-					first = false
-
-					//var full = page.path
-					var full = page.location
-
-					// handle recursion...
-					var parent_seen = 'seen' in state
-					var seen = state.seen = 
-						new Set(state.seen ?? [])
-					if(seen.has(full)
-							// nesting path recursion...
-							|| (full.length % (this.NESTING_RECURSION_TEST_THRESHOLD || 50) == 0
-								&& (pwpath.split(full).length > 3
-									&& new Set([
-											await page.find(),
-											await page.get('..').find(),
-											await page.get('../..').find(),
-										]).size == 1
-									// XXX HACK???
-									|| pwpath.split(full).length > (this.NESTING_DEPTH_LIMIT || 20)))){
-						if(recursive == null){
-							console.warn(
-								`@${key}(..): ${
-									seen.has(full) ?
-										'direct'
-										: 'depth-limit'
-								} recursion detected:`, full, seen)
-							yield page.get(page.RECURSION_ERROR).parse() 
-							continue }
-						// have the 'recursive' arg...
-						yield base.parse(recursive, state) 
-						continue }
-					seen.add(full)
-
-					// load the included page...
-					var res = await handler.call(page, full, state)
-					depends.add(full)
-					res = strquotes ?
-						res
-							.replace(/["']/g, function(c){
-								return '%'+ c.charCodeAt().toString(16) })
-						: res
-
-					// NOTE: we only track recursion down and not sideways...
-					seen.delete(full)
-					if(!parent_seen){
-						delete state.seen }
-
-					yield res } }),
 		// NOTE: the main difference between this and @include is that 
 		// 		this renders the src in the context of current page while 
 		// 		include is rendered in the context of its page but with
@@ -1671,30 +1643,48 @@ module.parser = {
 			['src', 'recursive', 'join', 
 				['s', 'strict']],
 			//['src'],
-			async function*(args, body, state){
+			function(page, args, body, state){
 				var that = this
-				yield* this.__parser__.macros.include.call(this, 
-					'source',
-					args, body, state, 'sources', 
-					async function(src, state){
-						//return that.parse(that.get(src).raw, state) }) }),
-						return that.parse(this.raw, state) }) }),
+				return this.macros['include'].call(this, 
+					page, args, body, state,
+					function(page, text, state){
+						return that.expand(page, text, state) }) }),
 
 		// Load macro and slot definitions but ignore the page text...
 		//
 		// NOTE: this is essentially the same as @source(..) but returns ''.
 		// XXX revise name...
-		// XXX UPDATE...
 		load: Macro(
-			['src', ['strict']],
-			async function*(args, body, state){
+			['src'],
+			function(args, body, state){
 				var that = this
-				yield* this.__parser__.macros.include.call(this, 
-					'load',
-					args, body, state, 'sources', 
-					async function(src, state){
-						await that.parse(this.raw, state) 
+				return Promise.awaitOrRun(
+					this.macros['include'].call(this, page, args, body, state),
+					function(){
 						return '' }) }),
+
+		// XXX a naive version, see ._quote(..) for reference...
+		quote: Macro(
+			['src', 'join', 'filter'],
+			quoting(
+			function(page, args, body, state){
+				var res = 
+					(args.src ?
+							page.get(args.src).raw
+						: body ?
+							body
+						: [])
+				return Promise.awaitOrRun(
+					res,
+					function(res){
+						// filter...
+						if(args.filter){
+							// XXX
+						}
+						// XXX join...
+						// XXX
+						return res }) })),
+
 		//
 		// 	@quote(<src>)
 		//
@@ -1717,7 +1707,7 @@ module.parser = {
 		// XXX need a way to escape macros -- i.e. include </quote> in a quoted text...
 		// XXX should join/else be sub-tags???
 		// XXX UPDATE...
-		quote: Macro(
+		_quote: Macro(
 			['src', 'filter', 'text', 'join', 'else',
 				['s', 'expandactions', 'strict']],
 			async function*(args, body, state){
@@ -1827,6 +1817,7 @@ module.parser = {
 			var filters = state.quote_filters = 
 				state.quote_filters ?? []
 			filters.splice(filters.length, 0, ...Object.keys(args)) },
+		//*/
 
 		// 	
 		// 	<macro src=<url>> .. </macro>

v3/pwiki/test/parser.js

@@ -17,6 +17,7 @@ module.exports.PAGES = {
 	'/page': 'Page',
 	'/async/page': Promise.resolve('Page'),
 	'/includePage': '@include(/page)',
+	'/isolated': '@slot(slot original)'
 }
 
 var P = 
@@ -50,6 +51,7 @@ module.exports.P = {
 //---------------------------------------------------------------------
 
 test.Setups({
+	// general parser...
 	empty: function(assert){
 		return { code: [ '', '' ] }},
 
@@ -141,11 +143,7 @@ test.Setups({
 		return {code: [
 			'<slot slot>moo<slot slot>[[ <content/> ]]</slot></slot>',
 			'[[ moo ]]' ]} },
-
-	/* XXX SHOWN_HIDDEN
-	// XXX these need to be revised...
-	// 		...do we actually need hidden/shown???
-	//
+	// shown/hidden..
 	slot_shown: function(assert){
 		var ins = this.slot_value(assert).code
 		var expect = ins.pop()
@@ -170,9 +168,8 @@ test.Setups({
 		var expect = ins.pop()
 		return {code: [
 			...ins.map(function(i){
-				return i +'@slot(slot shown)' }),
-			'' ]} },
-	//*/
+				return i +' | @slot(slot shown)' }),
+			' | value' ]} },
 	
 	// XXX var...
 	// XXX
@@ -181,7 +178,7 @@ test.Setups({
 	// XXX
 
 	// include...
-	// XXX these do not play well with modifiers...
+	// XXX see inside...
 	include: function(assert, path='/blank', expected){
 		return { 
 			page: P,
@@ -190,18 +187,37 @@ test.Setups({
 				// XXX for some reason this does not parse int a macro....
 				//'<include /blank />', 
 				`<include "${path}" />`, 
-					expected 
-						?? P.get(path).raw,
+				`<include src="${path}" />`, 
+
+				expected 
+					?? P.get(path).raw,
 			],
 		}},
 	include_page: function(assert){
 		return this.include(assert, '/page') },
 	include_include_page: function(assert){
 		return this.include(assert, '/includePage', 'Page') },
-	/* XXX parser.resolve(..) does not handle promises correctly yet...
 	include_async: function(assert){
-		return this.include(assert, '/async/page') },
-	//*/
+		return this.include(assert, '/async/page', 'Page') },
+	// islotaed...
+	include_nonisolated: function(assert){
+		return {
+			page: P,
+			code:[
+				'@include("/isolated") @slot(slot overloaded)',
+					'overloaded ',
+			],
+		} },
+	include_isolated: function(assert){
+		return {
+			page: P,
+			code:[
+				'@include(isolated /isolated) @slot(slot overloaded)',
+					'original overloaded',
+			],
+		} },
+	// XXX recursion...
+	// XXX
 })
 
 
@@ -234,7 +250,7 @@ test.Modifiers({
 
 
 test.Tests({
-	parse: function(assert, state){
+	parse: async function(assert, state){
 		var {page, code, st} = state
 		page ??= {}
 		st ??= {}
@@ -247,7 +263,7 @@ test.Tests({
 			var p = serialize.partialDeepCopy(page)
 			var s = serialize.partialDeepCopy(st)
 			assert(
-				(res = parser.parse(
+				(res = await parser.parse(
 						p,
 						input,
 						s))