added chunked iterators to util.js + some refactoring...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>

ui (gen4)/features/filesystem.js

@@ -765,11 +765,18 @@ var FileSystemLoaderActions = actions.Actions({
 	checkIndex: ['File/Check index consistency',
 		core.doc`Check index consistency...
 
+		Check currently loaded index for missing references and fix them
+		if found.
+
 		This will:
 			- remove references to non-existing preview images (image.preview)
 			- remove references to non-existing .path (image.path)
 			- if .path removed, set to largest available preview
 
+
+		NOTE: currently this is disabled for merged indexes, need to load
+			and check individually...
+
 		`,
 		function(logger){
 			var that = this
@@ -786,11 +793,15 @@ var FileSystemLoaderActions = actions.Actions({
 					.forEach(function(gid){ 
 						logger.emit('queued', gid)})
 
+			// XXX get this from config...
+			var chunk_size = 50
 
-			/* // XXX this version of the code blocks the ui until it's 
-			//		done...
 			return this.images
-				.map(function(gid, image){
+				.map(function(gid, image){ 
+					return [gid, image] })
+				.mapChunks(chunk_size, function(e){
+					var gid = e[0]
+					var image = e[1]
 					var updated = false
 
 					var previews = image.preview || {}
@@ -808,57 +819,8 @@ var FileSystemLoaderActions = actions.Actions({
 
 					return updated ? gid : []
 				})
-				.flat() 
+				.then(function(res){
-			/*/
+					return res.flat() })
-			var chunk_size = 50
-			return new Promise(function(resolve, reject){
-				var res = []
-				that.images
-					.reduce(function(res, e, gid){
-						var c = res.slice(-1)[0]
-						c.length < chunk_size ?
-							c.push([gid, e])
-							: res.push([[gid, e]])
-						return res
-					}, [[]])
-					.map(function(chunk, i, chunks){
-						setTimeout(function(){
-							res.push(chunk
-								// NOTE: all this complication with promises is
-								// 		needed to let the ui a chance to show 
-								// 		progress...
-								// NOTE: if a better way is found this is the 
-								// 		only part needed, just iterate over
-								// 			return this.images
-								// 				.map(function(gid, image){
-								// 					// place the func body here...
-								// 					...
-								// 				})
-								.map(function(v){
-									var gid = v[0]
-									var image = v[1]
-									var updated = false
-
-									var previews = image.preview || {}
-									Object.entries(previews)
-										.forEach(function(p){
-											!fse.existsSync(image.base_path +'/'+ p[1])
-												&& (updated = true)
-												&& (delete previews[p[0]]) })
-
-									!fse.existsSync(image.base_path +'/'+ image.path)
-										&& (updated = true)
-										&& (delete image.path)
-
-									logger && logger.emit('done', gid)
-
-									return updated ? gid : []
-								})) 
-
-							i >= chunks.length-1
-								&& resolve(res.flat(Infinity))
-						}, 0) }) }) 
-			//*/
 		}],
 })
 

ui (gen4)/lib/util.js

@@ -198,6 +198,84 @@ Array.prototype.sortAs = function(other){
 
 
 
+// Equivalent to .map(..) / .filter(..) / .reduce(..) / .forEach(..) that
+// process the contents in chunks asynchronously...
+//
+//	.mapChunks(func)
+//	.mapChunks(chunk_size, func)
+//		-> promise(list)
+//	
+//	.filterChunks(func)
+//	.filterChunks(chunk_size, func)
+//		-> promise(list)
+//	
+//	.reduceChunks(func, res)
+//	.reduceChunks(chunk_size, func, res)
+//		-> promise(res)
+//	
+//
+var makeChunkIter = function(iter, wrapper){
+	wrapper = wrapper
+		|| function(res, func, array, e){
+			return func.call(this, e[1], e[0], array) }
+	return function(size, func, ...rest){
+		var that = this
+		var args = [...arguments]
+		size = args[0] instanceof Function ? 
+			(this.chunk_size || 50)
+			: args.shift()
+		func = args.shift()
+		rest = args
+		var res = []
+		var _wrapper = wrapper.bind(this, res, func, this)
+
+		return new Promise(function(resolve, reject){
+			that
+				// split the array into chunks...
+				.reduce(function(res, e, i){
+					var c = res.slice(-1)[0]
+					c.length >= size ?
+						// initial element in chunk...
+						res.push([[i, e]])
+						// rest...
+						: c.push([i, e])
+					return res
+				}, [[]])
+				// go through each chunk async...
+				.forEach(function(chunk, i, chunks){
+					setTimeout(function(){
+						res.push(
+							// NOTE: all this complication with promises is
+							// 		needed to let the ui a chance to show 
+							// 		progress...
+							// NOTE: if a better way is found this is the 
+							// 		only part needed, just iterate over
+							// 			return this.images
+							// 				.map(function(gid, image){
+							// 					// place the func body here...
+							// 					...
+							// 				})
+							chunk[iter](_wrapper, ...rest))
+
+						i >= chunks.length-1
+							&& resolve(res.flat(2))
+					}, 0) }) }) } }
+
+Array.prototype.chunk_size = 50 
+Array.prototype.mapChunks = makeChunkIter('map')
+Array.prototype.filterChunks = makeChunkIter('map', 
+	function(res, func, array, e){
+		return !!func.call(this, e[1], e[0], array) ? [e[1]] : [] })
+Array.prototype.reduceChunks = makeChunkIter('reduce',
+	function(total, func, array, res, e){
+		return func.call(this, 
+			total.length > 0 ? 
+				total.pop() 
+				: res, 
+			e[1], e[0], array) })
+
+
+
 //---------------------------------------------------------------------
 // Set...