reqorked logging and progress indication...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>

ui (gen4)/features/filesystem.js

@@ -491,6 +491,7 @@ var FileSystemLoaderActions = actions.Actions({
 	//
 	// Returns: Images object
 	//
+	// XXX revise logging...
 	getImagesInPath: ['- File/',
 		function(path, read_stat, skip_preview_search, logger){
 			if(path == null){
@@ -503,24 +504,42 @@ var FileSystemLoaderActions = actions.Actions({
 				this.config['image-file-skip-previews']
 				: skip_preview_search
 
-			// XXX get a logger...
 			logger = logger || this.logger
-			//logger = logger && logger.push('getImagesInPath')
 
 			var that = this
 			path = util.normalizePath(path)
 
+			// progress...
+			// XXX this does not appear to run while glob(..) is running...
+			var found = []
+			var update_interval
+			if(logger){
+				that.showProgress 
+					&& that.showProgress(logger.path)
+				update_interval = setInterval(function(){
+					found.length > 0
+						&& logger.emit('found', found) 
+						&& (found = []) }, 150) }
+
 			// get the image list...
 			return new Promise(function(resolve, reject){
 				glob(path + '/'+ that.config['image-file-pattern'], {
 						stat: !!read_stat,
 						strict: false,
 					})
+					.on('match', function(e){ found.push(e) })
 					.on('error', function(err){
+						update_interval
+							&& clearInterval(update_interval)
 						console.error(err)
-						reject(err)
+						reject(err) })
-					})
 					.on('end', function(lst){ 
+						update_interval
+							&& clearInterval(update_interval)
+						logger && found.length > 0
+							&& logger.emit('found', found)
+							&& (found = [])
+
 						// XXX might be a good idea to make image paths relative to path...
 						//lst = lst.map(function(p){ return pathlib.relative(base, p) })
 						// XXX do we need to normalize paths after we get them from glob??
@@ -543,8 +562,7 @@ var FileSystemLoaderActions = actions.Actions({
 								img.size = stat.size
 
 								// XXX do we need anything else???
-							})
+							}) }
-						}
 
 						// pass on the result...
 						resolve(imgs)
@@ -558,9 +576,7 @@ var FileSystemLoaderActions = actions.Actions({
 				return !skip_preview_search ? 
 					//that.getPreviews('all', path, imgs)
 					that.getPreviews('all', index_path, imgs)
-					: imgs 
+					: imgs }) }],
-			})
-		}],
 
 	// Load images...
 	//
@@ -593,6 +609,7 @@ var FileSystemLoaderActions = actions.Actions({
 					logger)
 				// load the data...
 				.then(function(imgs){
+					logger && logger.emit('loaded', imgs.keys()) 
 					return that.loadOrRecover({
 							images: imgs,
 							data: data.Data.fromArray(imgs.keys()),
@@ -604,10 +621,7 @@ var FileSystemLoaderActions = actions.Actions({
 							}
 						})
 						.then(function(){
-							that.markChanged('none')
+							that.markChanged('none') }) }) }],
-						})
-				})
-		}],
 
 	// Load images to new ribbon...
 	//
@@ -650,6 +664,8 @@ var FileSystemLoaderActions = actions.Actions({
 					logger)
 				// load the data...
 				.then(function(imgs){
+					logger && logger.emit('loaded', imgs.keys()) 
+
 					that.clearLoaction()
 
 					var d = that.data
@@ -699,19 +715,19 @@ var FileSystemLoaderActions = actions.Actions({
 				return ['loadIndex', 'loadImages'].includes(this.location.load) 
 					|| 'disabled' }, },
 		function(path, logger){
+			var that = this
 			path = path || this.location.path
 
 			if(path == null){
 				return
 			}
 
-			var that = this
 			logger = logger || this.logger
 			logger = logger && logger.push('Load new images')
 			path = util.normalizePath(path)
 
 			// cache the loaded images...
-			var loaded = this.images.map(function(gid, img){ return img.path })
+			var loaded = new Set(this.images.map(function(gid, img){ return img.path }))
 			//var base_pattern = RegExp('^'+path)
 
 			return this.getImagesInPath(
@@ -721,25 +737,47 @@ var FileSystemLoaderActions = actions.Actions({
 					logger)
 				// load the data...
 				.then(function(imgs){
+					var added = []
+					var skipped = []
+					var progress = function(){
+						skipped.length > 0
+							&& logger.emit('skipped', skipped)
+							&& (skipped = [])
+						added.length > 0
+							&& logger.emit('done', added)
+							&& (added = []) }
+
 					// remove the images we already have loaded...
+					var t = Date.now()
 					imgs.forEach(function(gid, img){
+						// XXX this does not let the browser update progress...
+						Date.now() - t > 200
+							&& (t = Date.now())
+							&& progress()
 						// NOTE: we do not need to normalize anything as
 						// 		both the current path and loaded paths 
 						// 		came from the same code...
 						// XXX is this good enough???
 						// 		...might be a good idea to compare absolute
 						// 		paths...
-						if(loaded.indexOf(img.path) >= 0){
+						if(loaded.has(img.path)){
 							delete imgs[gid] 
-						}	
+							skipped.push(gid)
-					})
+						} else {
+							added.push(gid) } })
+
+					// finalize progress...
+					if(logger){
+						skipped.length > 0
+							&& logger.emit('skipped', skipped)
+						added.length > 0
+							&& logger.emit('done', added) }
 
 					// nothing new...
 					if(imgs.length == 0){
 						// XXX
 						logger && logger.emit('loaded', [])
-						return imgs
+						return imgs }
-					}
 
 					// XXX
 					logger && logger.emit('queued', imgs)
@@ -863,39 +901,47 @@ var FileSystemLoaderActions = actions.Actions({
 			logger = logger && logger.push('Check missing')
 
 			logger 
-				&& this.images
+				&& logger.emit('queued', this.images.keys())
-					.forEach(function(gid){ 
-						logger.emit('queued', gid)})
 
 			var chunk_size = '100C'
+			var removed = []
 
 			return this.images
 				.map(function(gid, image){ 
 					return [gid, image] })
-				.mapChunks(chunk_size, function([gid, image]){
+				.mapChunks(chunk_size, [
-					var updated = false
+					function([gid, image]){
+						var updated = false
 
-					image.path 
+						image.path 
-						&& !fse.existsSync(image.base_path +'/'+ image.path)
+							&& !fse.existsSync(image.base_path +'/'+ image.path)
-						&& (updated = true)
+							&& (updated = true)
-						&& logger && rem_logger.emit('queued', gid)
+							&& logger 
+								&& removed.push(gid)
 
-					logger && logger.emit('done', gid)
+						return updated ? gid : [] },
-
+					// do the logging per chunk...
-					return updated ? gid : []
+					function(chunk, res){
-				})
+						logger 
+							&& logger.emit('done', chunk.map(function([gid]){ return gid })) 
+							&& rem_logger.emit('queued', removed)
+							&& (removed = []) }])
 				.then(function(res){
 					res = res.flat()
-					if(res.length > 0){
+					return res.length > 0 ?
-						logger && rem_logger.emit('queued', 'data cleanup')
+						res
-						// clear images...
+							.mapChunks(chunk_size, [
-						res.forEach(function(gid){
+								// clear images...
-							delete that.images[gid]
+								function(gid){
-							logger && rem_logger.emit('done', gid) })
+									delete that.images[gid] }, 
-						// clear data...
+								// log...
-						that.data.clear(res)
+								function(chunk){
-						logger && rem_logger.emit('done', 'data cleanup') }
+									logger && rem_logger.emit('done', chunk) }])
-					return res }) }],
+							// clear data...
+							.then(function(){
+								that.data.clear(res) 
+								return res })
+						: res }) }],
 
 
 	// XXX EXPERIMENTAL...

ui (gen4)/features/ui-progress.js

@@ -20,6 +20,8 @@ var ProgressActions = actions.Actions({
 	config: {
 		'progress-fade-duration': 200,
 		'progress-done-delay': 1000,
+
+		'progress-update-min': 200,
 	},
 
 	// Progress bar widget...
@@ -41,20 +43,56 @@ var ProgressActions = actions.Actions({
 	// 	.showProgress('text', '+1')
 	// 	.showProgress('text', '+0', '+1')
 	//
+	// 	.showProgress(logger)
+	//
 	//
 	// XXX add message to be shown...
 	// XXX should we report errors and stoppages??? (error state??)
 	// XXX multiple containers...
 	// XXX shorten the nested css class names...
 	// XXX revise styles...
+	__progress_cache: null,
 	showProgress: ['- Interface/Show progress bar...',
 		function(text, value, max){
-			var viewer = this.dom
 			var that = this
+			var viewer = this.dom
 
 			var msg = text instanceof Array ? text.slice(1).join(': ') : null
 			text = text instanceof Array ? text[0] : text
 
+			// make sure we do not update too often...
+			if(value != 'close'){
+				var cache = (this.__progress_cache = this.__progress_cache || {})
+				cache = cache[text] = cache[text] || {}
+
+				var updateValue = function(name, value){
+					var v = cache[name] || 0
+					return (cache[name] = 
+						value != null ? 
+							(typeof(value) == typeof('str') && /[+-][0-9]+/.test(value) ? 
+								v + parseInt(value)
+								: parseInt(value))
+							: v) }
+
+				value = updateValue('value', value)
+				max = updateValue('max', max)
+
+				// update not due yet...
+				if('timeout' in cache){
+					cache.update = true
+					return 
+
+				// set next update point and continue...
+				} else {
+					delete cache.update
+					cache.timeout = setTimeout(
+						function(){
+							var cache = that.__progress_cache[text] || {}
+							delete cache.timeout
+							cache.update
+								&& that.showProgress(text) }, 
+						this.config['progress-update-min'] || 200) } }
+
 			// container...
 			var container = viewer.find('.progress-container')
 			container = container.length == 0 ?
@@ -68,8 +106,8 @@ var ProgressActions = actions.Actions({
 			// close action...
 			if(value == 'close'){
 				widget.trigger('progressClose')
-				return
+				return }
-			}
+
 			widget = widget.length == 0 ?
 				$('<div/>')
 					.addClass('progress-bar')
@@ -87,6 +125,10 @@ var ProgressActions = actions.Actions({
 					.on('progressClose', function(){ 
 						widget
 							.fadeOut(that.config['progress-fade-duration'] || 200, function(){
+								var cache = (that.__progress_cache || {})
+								cache[text].timeout 
+									&& clearTimeout(cache[text].timeout)
+								delete cache[text]
 								$(this).remove() }) })
 					.appendTo(container)
 				: widget
@@ -99,26 +141,11 @@ var ProgressActions = actions.Actions({
 			var bar = widget.find('progress')
 			var state = widget.find('.progress-details')
 
-			// XXX stub???
-			// normalize max and value...
-			max = max != null ? 
-					(typeof(max) == typeof('str') && /[+-][0-9]+/.test(max) ? 
-						parseInt(bar.attr('max') || 0) + parseInt(max)
-					: parseInt(max))
-				: bar.attr('max')
-			value = value != null ? 
-					(typeof(value) == typeof('str') && /[+-][0-9]+/.test(value) ? 
-						parseInt(bar.attr('value') || 0) + parseInt(value)
-					: parseInt(value))
-				: bar.attr('value')
-
 			// format the message...
-			// XXX should we add a message after this????
 			msg = msg ? ': '+msg : ''
 			msg = ' '+ msg 
-				//+ (value && value >= (max || 0) ? ' ('+value+' done)' 
 				+ (value && value >= (max || 0) ? ' (done)' 
-					: value && max && value != max ? ' ('+ value +' of '+ max +')'
+					: max && value != max ? ' ('+ (value || 0) +' of '+ max +')'
 					: '...')
 
 			// update widget...
@@ -129,48 +156,59 @@ var ProgressActions = actions.Actions({
 			state.text(msg)
 
 			// auto-close...
-			// XXX make this optional... 
 			if(value && value >= (max || 0)){
 				widget.attr('close-timeout', 
 					JSON.stringify(setTimeout(function(){ 
 						widget.trigger('progressClose') 
-					}, this.config['progress-done-delay'] || 1000)))
+					}, this.config['progress-done-delay'] || 1000))) }
-			}
 
 			// XXX force the browser to render...
 			//bar.hide(0).show(0)
-
-			// XXX what should we return??? (state, self, controller?)
 		}],
 
 	// handle logger progress...
 	// XXX revise...
 	handleLogItem: ['- System/',
 		function(path, status, ...rest){
-			var msg = this.message
+			var msg = path.join(': ')
+			var l = (rest.length == 1 && rest[0] instanceof Array) ?
+				rest[0].length
+				: rest.length
+
+			// XXX should we move these to a more accessible spot???
+			var add = [
+				'added',
+				'queued',
+				'found',
+			]
+			var done = [
+				'loaded',
+				'done',
+				'written',
+				'index',
+			]
+			var skipped = [
+				'skipping',
+				'skipped',
+				'removed',
+			]
 
 			// report progress...
 			// XXX HACK -- need meaningful status...
-			if(status == 'queued' 
+			if(add.includes(status)){
-					|| status == 'found'){
+				this.showProgress(msg, '+0', '+'+l)
-				this.showProgress(msg || ['Progress', status], '+0', '+'+rest.length)
 
-			} else if(status == 'loaded' || status == 'done' || status == 'written' 
+			} else if(done.includes(status)){
-					|| status == 'index'){
+				this.showProgress(msg, '+'+l)
-				this.showProgress(msg || ['Progress', status], '+'+rest.length)
 
-			} else if(status == 'skipping' || status == 'skipped'){
+			} else if(skipped.includes(status)){
 				// XXX if everything is skipped the indicator does not 
 				// 		get hidden...
-				//this.showProgress(msg || ['Progress', status], '+0', '-1')
+				this.showProgress(msg, '+'+l)
-				this.showProgress(msg || ['Progress', status], '+'+rest.length)
 
 			// XXX STUB...
 			} else if(status == 'error' ){
-				this.showProgress(['Error'].concat(msg), '+0', '+'+rest.length)
+				this.showProgress(['Error'].concat(msg), '+0', '+'+l)
-				//console.log(msg ? 
-				//	'    '+ msg.join(': ') + ':' 
-				//	: '', ...arguments) 
 			}
 		}],
 })

ui (gen4)/imagegrid/file.js

@@ -244,7 +244,7 @@ function(list){
 var groupByKeyword = 
 module.groupByKeyword = 
 function(list, from_date, logger){
-	logger = logger && logger.push('Grouping by keyword')
+	//logger = logger && logger.push('Grouping by keyword')
 
 	var index = {}
 	var queued = 0
@@ -312,8 +312,8 @@ function(list, from_date, logger){
 
 	// remove the flags...
 	for(var k in index){
-		index[k] = index[k].map(function(e){ return e[1] })
+		index[k] = index[k]
-	}
+			.map(function(e){ return e[1] }) }
 
 	logger && logger.emit('files-queued', queued, index)
 

ui (gen4)/lib/util.js

@@ -196,17 +196,26 @@ Array.prototype.sortAs = function(other){
 //
 //	.mapChunks(func)
 //	.mapChunks(chunk_size, func)
+//	.mapChunks([item_handler, chunk_handler])
+//	.mapChunks(chunk_size, [item_handler, chunk_handler])
 //		-> promise(list)
 //	
 //	.filterChunks(func)
 //	.filterChunks(chunk_size, func)
+//	.filterChunks([item_handler, chunk_handler])
+//	.filterChunks(chunk_size, [item_handler, chunk_handler])
 //		-> promise(list)
 //	
 //	.reduceChunks(func, res)
 //	.reduceChunks(chunk_size, func, res)
+//	.reduceChunks([item_handler, chunk_handler], res)
+//	.reduceChunks(chunk_size, [item_handler, chunk_handler], res)
 //		-> promise(res)
 //
 //
+//	chunk_handler(chunk, result, offset)
+//
+//
 // chunk_size can be:
 // 	20			- chunk size
 // 	'20'		- chunk size
@@ -223,7 +232,8 @@ var makeChunkIter = function(iter, wrapper){
 	return function(size, func, ...rest){
 		var that = this
 		var args = [...arguments]
-		size = args[0] instanceof Function ? 
+		size = (args[0] instanceof Function 
+				|| args[0] instanceof Array) ? 
 			(this.CHUNK_SIZE || 50)
 			: args.shift()
 		size = typeof(size) == typeof('str') ?
@@ -232,31 +242,38 @@ var makeChunkIter = function(iter, wrapper){
 				 	Math.round(this.length / (parseInt(size) || 1)) || 1
 				: parseInt(size))
 			: size
+		var postChunk
 		func = args.shift()
+		;[func, postChunk] = func instanceof Array ? func : [func]
 		rest = args
 		var res = []
 		var _wrapper = wrapper.bind(this, res, func, this)
 
 		return new Promise(function(resolve, reject){
-			var next = function(chunks){
+				var next = function(chunks){
-				setTimeout(function(){
+					setTimeout(function(){
-					res.push(
+						var chunk, val
-						chunks.shift()[iter](_wrapper, ...rest))
+						res.push(
-					// stop condition...
+							val = (chunk = chunks.shift())[iter](_wrapper, ...rest))
-					chunks.length == 0 ?
+						postChunk
-						resolve(res.flat(2))
+							&& postChunk.call(that, 
-						: next(chunks) }, 0) }
+								chunk.map(function([i, v]){ return v }), 
-			next(that
+								val,
-				// split the array into chunks...
+								chunk[0][0])
-				.reduce(function(res, e, i){
+						// stop condition...
-					var c = res.slice(-1)[0]
+						chunks.length == 0 ?
-					c.length >= size ?
+							resolve(res.flat(2))
-						// initial element in chunk...
+							: next(chunks) }, 0) }
-						res.push([[i, e]])
+				next(that
-						// rest...
+					// split the array into chunks...
-						: c.push([i, e])
+					.reduce(function(res, e, i){
-					return res }, [[]]))
+						var c = res.slice(-1)[0]
-		}) } }
+						c.length >= size ?
+							// initial element in chunk...
+							res.push([[i, e]])
+							// rest...
+							: c.push([i, e])
+						return res }, [[]])) }) } }
 
 Array.prototype.CHUNK_SIZE = 50 
 Array.prototype.mapChunks = makeChunkIter('map')