From 7122c619f417115e5ef4a4cc3505879de8c91442 Mon Sep 17 00:00:00 2001
From: "Alex A. Naanou" <alex.nanou@gmail.com>
Date: Wed, 2 Dec 2020 18:49:45 +0300
Subject: [PATCH] reworked queued action + now .cacheMetadata(..) is queued...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>
---
 Viewer/features/core.js     |  85 ++++++++---
 Viewer/features/examples.js |  20 ++-
 Viewer/features/metadata.js | 116 ++++++++-------
 Viewer/features/sharp.js    | 290 ++++++++++++------------------------
 Viewer/package-lock.json    |   6 +-
 Viewer/package.json         |   2 +-
 6 files changed, 251 insertions(+), 268 deletions(-)

diff --git a/Viewer/features/core.js b/Viewer/features/core.js
index c97fd35d..b6684f90 100755
--- a/Viewer/features/core.js
+++ b/Viewer/features/core.js
@@ -2520,6 +2520,7 @@ function(func){
 // 		...would also be nice to automate this via a chunk iterator so
 // 		as not to block...
 // XXX handle errors... (???)
+// XXX revise logging and logger passing...
 var queuedAction = 
 module.queuedAction =
 function(name, func){
@@ -2543,25 +2544,69 @@ function(name, func){
 		}) }
 
 
+//
+//
+//	queueHandler(name[, opts][, arg_handler], func)
+//		-> action
+//
+//
+//	arg_handler(...args)
+//		-> [items, ...args]
+//
+//
+//	action(items, ...args)
+//		-> promise
+//
+//	action('sync', items, ...args)
+//		-> promise
+//
+//
+//	func(item, ...args)
+//		-> res
+//
+//
+// XXX should 'sync' set .sync_start or just .runTask(..)
+// XXX check if item is already in queue...
 var queueHandler =
 module.queueHandler =
 function(name, func){
 	var args = [...arguments]
 	func = args.pop()	
+	var arg_handler = 
+		typeof(args.last()) == 'function' 
+			&& args.pop()
 	var [name, opts] = args
 
 	return object.mixin(
 		Queued(function(items, ...args){
 			var that = this
-			return new Promise(function(resolve, reject){
-				var q = that.queue(name,
-						Object.assign(
-							{},
-							opts || {},
-							{ handler: function(item){
+			// sync start...
+			if(arguments[0] == 'sync'){
+				var [sync, items, ...args] = arguments }
+			// prep queue...
+			var q = that.queue(name,
+					Object.assign(
+						{},
+						opts || {},
+						{ handler: function(item){
 								return func.call(that, item, ...args) } }))
-				q.push(...(items instanceof Array ? items : [items]))
-				q.then(resolve, reject) }) }),
+			sync 
+				&& (q.sync_start = true)
+			// pre-process args...
+			arg_handler
+				&& ([items, ...args] = 
+					arg_handler.call(this, q, items, ...args))
+			// fill the queue...
+			q.push(...(items instanceof Array ? items : [items]))
+			// make a promise...
+			var res = new Promise(function(resolve, reject){
+				q.then(resolve, reject) }) 
+			// sync start...
+			// NOTE: we need to explicitly .start() here because the 
+			// 		queue could be waiting for a timeout
+			sync
+				&& q.start()
+			return res }),
    		{
 			toString: function(){
 				return `core.queueHandler('${name}',\n\t${ 
@@ -2653,13 +2698,13 @@ var TaskActions = actions.Actions({
 	get queues(){
 		return (this.__queues = this.__queues || {}) },
 
-	// XXX revise signature...
+	// XXX revise logging and logger passing...
 	// XXX need better error flow...
 	queue: doc('Get or create a queue task',
 		doc`Get or create a queue task...
 
 			.queue(name)
-			.queue(name[, options][, logger])
+			.queue(name, options)
 				-> queue
 
 		If a queue with the given name already exits it will be returned 
@@ -2675,7 +2720,7 @@ var TaskActions = actions.Actions({
 
 		NOTE: for queue-specific options see ig-types/runner's Queue(..)
 		`,
-		function(name, options, logger){
+		function(name, options){
 			var that = this
 
 			var queue = this.queues[name]
@@ -2692,21 +2737,25 @@ var TaskActions = actions.Actions({
 						//	&& logger.emit('close')
 						delete that.queues[name] } }
 
-				var logger = logger || this.logger
+				options = options || {}
+				var logger = options.logger || this.logger
 				//logger = logger && logger.push(name)
 				logger = logger 
 					&& logger.push(name, {onclose: abort, quiet: !!options.quiet})
+				logger 
+					&& (options.logger = logger)
 
 				queue = this.queues[name] = 
 					runner.Queue(options || {})
 
 				// setup logging...
-				if(logger){
-					queue
-						.on('tasksAdded', function(evt, t){ logger.emit('added', t) })
-						.on('taskCompleted', function(evt, t){ logger.emit('done', t) }) 
-						.on('taskFailed', function(evt, t, err){ logger.emit('skipped', t, err) }) 
-					queue.logger = logger }
+				queue
+					.on('tasksAdded', function(evt, t){ 
+						this.logger && this.logger.emit('added', t) })
+					.on('taskCompleted', function(evt, t){ 
+						this.logger && this.logger.emit('done', t) }) 
+					.on('taskFailed', function(evt, t, err){ 
+						this.logger && this.logger.emit('skipped', t, err) }) 
 				// cleanup...
 				queue
 					.then(
diff --git a/Viewer/features/examples.js b/Viewer/features/examples.js
index d3a750b6..4168d496 100755
--- a/Viewer/features/examples.js
+++ b/Viewer/features/examples.js
@@ -280,7 +280,25 @@ var ExampleActions = actions.Actions({
 				this.exampleQueuedAction(timeout) } }],
 
 	exampleQueueHandlerAction: ['- Test/',
-		core.queueHandler('exampleQueueHandlerAction', {quiet: true}, 
+		core.queueHandler('exampleQueueHandlerAction', 
+			{quiet: true}, 
+			function(item, ...args){
+				console.log('Queue handler action!!', item, ...args)
+				return new Promise(function(resolve){
+					setTimeout(resolve, 100) }) })],
+	exampleQueueHandlerActionWArgs: ['- Test/',
+		core.queueHandler('exampleQueueHandlerActionWArgs', 
+			{quiet: true}, 
+			function(queue, from=0, to=100, ...args){
+				var items = []
+				var reverse = from > to
+				reverse
+					&& ([from, to] = [to+1, from+1])
+				for(var i=from; i<to; i++){
+					items.push(i) }
+				reverse
+					&& items.reverse()
+				return [items, ...args] },
 			function(item, ...args){
 				console.log('Queue handler action!!', item, ...args)
 				return new Promise(function(resolve){
diff --git a/Viewer/features/metadata.js b/Viewer/features/metadata.js
index af2a9a47..d1411643 100755
--- a/Viewer/features/metadata.js
+++ b/Viewer/features/metadata.js
@@ -115,69 +115,79 @@ var MetadataReaderActions = actions.Actions({
 
 		NOTE: also see: .cacheMetadata(..)
 		`,
-		core.queueHandler('Read image metadata', function(image, force){
-			var that = this
+		core.queueHandler('Read image metadata', 
+			function(image, force){
+				return [
+					images == 'all' ?
+							this.images.keys()
+						: images == 'loaded' ?
+							this.data.getImages('loaded')
+						: images,
+					force, 
+				] },
+			function(image, force){
+				var that = this
 
-			var gid = this.data.getImage(image)
-			var img = this.images && this.images[gid]
+				var gid = this.data.getImage(image)
+				var img = this.images && this.images[gid]
 
-			if(!image && !img){
-				return false }
+				if(!image && !img){
+					return false }
 
-			//var full_path = path.normalize(img.base_path +'/'+ img.path)
-			var full_path = this.getImagePath(gid)
+				//var full_path = path.normalize(img.base_path +'/'+ img.path)
+				var full_path = this.getImagePath(gid)
 
-			return new Promise(function(resolve, reject){
-				if(!force 
-						&& (img.metadata || {}).ImageGridMetadata == 'full'){
-					return resolve(img.metadata) }
+				return new Promise(function(resolve, reject){
+					if(!force 
+							&& (img.metadata || {}).ImageGridMetadata == 'full'){
+						return resolve(img.metadata) }
 
-				fs.readFile(full_path, function(err, file){
-					if(err){
-						return reject(err) }
-
-					// read stat...
-					if(!that.images[gid].birthtime){
-						var img = that.images[gid]
-						var stat = fs.statSync(full_path)
-						
-						img.atime = stat.atime
-						img.mtime = stat.mtime
-						img.ctime = stat.ctime
-						img.birthtime = stat.birthtime
-
-						img.size = stat.size
-					}
-
-					// read image metadata...
-					exiftool.metadata(file, function(err, data){
+					fs.readFile(full_path, function(err, file){
 						if(err){
-							reject(err)
+							return reject(err) }
 
-						} else if(data.error){
-							reject(data)
+						// read stat...
+						if(!that.images[gid].birthtime){
+							var img = that.images[gid]
+							var stat = fs.statSync(full_path)
+							
+							img.atime = stat.atime
+							img.mtime = stat.mtime
+							img.ctime = stat.ctime
+							img.birthtime = stat.birthtime
 
-						} else {
-							// convert to a real dict...
-							// NOTE: exiftool appears to return an array 
-							// 		object rather than an actual dict/object
-							// 		and that is not JSON compatible....
-							that.images[gid].metadata =
-								Object.assign(
-									// XXX do we need to update or overwrite??
-									that.images[gid].metadata || {},
-									data,
-									{
-										ImageGridMetadataReader: 'exiftool/ImageGrid',
-										// mark metadata as full read...
-										ImageGridMetadata: 'full',
-									})
-							that.markChanged 
-								&& that.markChanged('images', [gid]) }
+							img.size = stat.size
+						}
 
-						resolve(data) }) }) }) })],
+						// read image metadata...
+						exiftool.metadata(file, function(err, data){
+							if(err){
+								reject(err)
+
+							} else if(data.error){
+								reject(data)
+
+							} else {
+								// convert to a real dict...
+								// NOTE: exiftool appears to return an array 
+								// 		object rather than an actual dict/object
+								// 		and that is not JSON compatible....
+								that.images[gid].metadata =
+									Object.assign(
+										// XXX do we need to update or overwrite??
+										that.images[gid].metadata || {},
+										data,
+										{
+											ImageGridMetadataReader: 'exiftool/ImageGrid',
+											// mark metadata as full read...
+											ImageGridMetadata: 'full',
+										})
+								that.markChanged 
+									&& that.markChanged('images', [gid]) }
+
+							resolve(data) }) }) }) })],
 	readAllMetadata: ['File/Read all metadata',
-		'readMetadata: images.gids ...'],
+		'readMetadata: "all" ...'],
 
 	// XXX take image Metadata and write it to target...
 	writeMetadata: ['- Image/Set metadata data',
diff --git a/Viewer/features/sharp.js b/Viewer/features/sharp.js
index 5f7b1622..4390f9c4 100755
--- a/Viewer/features/sharp.js
+++ b/Viewer/features/sharp.js
@@ -376,7 +376,7 @@ var SharpActions = actions.Actions({
 												&& Math.max(m.width, m.height) < size)
 											|| (fit == 'outside'
 												&& Math.min(m.width, m.height) < size)){
-										logger && logger.emit('skipping', gid)
+										//logger && logger.emit('skipping', gid)
 										return }
 									// continue...
 									return img })
@@ -396,7 +396,7 @@ var SharpActions = actions.Actions({
 												fse.removeSync(to)
 											// skip...
 											} else {
-												logger && logger.emit('skipping', gid)
+												//logger && logger.emit('skipping', gid)
 												return } }
 
 										// write...
@@ -588,9 +588,8 @@ var SharpActions = actions.Actions({
 						Promise.reject('aborted')
 						: res.flat() }) })],
 
-	// XXX will this be better off as a queueHandler(..) ???
 	// XXX add support for offloading the processing to a thread/worker...
-	__cache_metadata_reading: null,
+	// XXX revise logging and logger passing...
 	cacheMetadata: ['- Sharp|Image/',
 		core.doc`Cache metadata
 
@@ -638,203 +637,110 @@ var SharpActions = actions.Actions({
 		NOTE: this will effectively update metadata format to the new spec...
 		NOTE: for info on full metadata format see: .readMetadata(..)
 		`,
-		core.sessionTaskAction('cacheMetadata', function(ticket, images, logger){
-			var that = this
+		core.queueHandler('Cache image metadata', 
+			{quiet: true}, 
+			// parse args...
+			function(queue, image, logger){
+				var force = false
+				if(image === true){
+					var [force, image, logger] = arguments }
+				return [
+					image == 'all' ?
+							this.images.keys()
+						: image == 'loaded' ?
+							this.data.getImages('loaded')
+						: (image || 'current'),
+					force, 
+					// XXX
+					logger || this.logger,
+				] },
+			function(image, force, logger){
+				var that = this
 
-			// setup runtime interactions...
-			//
-			// NOTE: we will resolve the ticket when we are fully done
-			// 		and not on stop...
-			var STOP = false
-			ticket
-				.onmessage('stop', function(){
-					STOP = true }) 
-				.then(function(){
-					// close progress bar...
-					// NOTE: if we have multiple tasks let the last one 
-					// 		close the progress bar...
-					if(that.tasks.titled(ticket.title).length == 0){
-						logger 
-							&& logger.emit('close') }
-					that.off('clear', on_close)
-					// cleanup...
-					delete that.__cache_metadata_reading })
-			// clear the progress bar for the next session...
-			var on_close
-			this.one('clear', on_close = function(){
-				logger && logger.emit('close') })
+				// XXX cache the image data???
+				var gid = this.data.getImage(image)
+				var img = this.images[gid]
+				var path = img && that.getImagePath(gid)
 
-			// universal task abort...
-			// NOTE: this will abort all the tasks of this type...
-			var abort = function(){
-				that.tasks.stop(ticket.title) }
+				// XXX
+				//var base_path = that.location.load == 'loadIndex' ?
+				//	null
+				//	: tmp
+				//var base_path = img && img.base_path
+				var base_path
+		
+				// skip...
+				if(!(img && path
+						&& (force
+							// high priority must be preset...
+							|| (img.orientation == null
+								&& img.flipped == null)
+							// update metadata...
+							|| (img.metadata || {}).ImageGridMetadata == null))){
+					return }
 
-			var CHUNK_SIZE = 4
+				// XXX handle/report errors...
+				return sharp(that.getImagePath(gid))
+					.metadata()
+					.then(function(metadata){
+						// no metadata...
+						if(metadata == null){
+							return }
 
-			// handle logging and processing list...
-			// NOTE: these will maintain .__cache_metadata_reading helping 
-			// 		avoid processing an image more than once at the same 
-			// 		time...
-			var done = function(gid, msg){
-				logger && logger.emit(msg || 'done', gid)
-				if(that.__cache_metadata_reading){
-					that.__cache_metadata_reading.delete(gid) }
-				return gid }
-			var skipping = function(gid){
-				return done(gid, 'skipping') }
+						var o = normalizeOrientation(metadata.orientation)
+						;(force || img.orientation == null)
+							// NOTE: we need to set orientation to something
+							// 		or we'll check it again and again...
+							&& (img.orientation = o.orientation || 0)
+						;(force || img.flipped == null)
+							&& (img.flipped = o.flipped)
 
-			var force = false
-			if(images === true){
-				force = true
-				images = null
+						// mark metadata as partially read...
+						// NOTE: this will intentionally overwrite the 
+						// 		previous reader mark/mode...
+						img.metadata =
+							Object.assign(
+								img.metadata || {}, 
+								{ 
+									ImageGridMetadataReader: 'sharp/exif-reader/ImageGrid',
+									// mark metadata as partial read...
+									// NOTE: partial metadata will get reread by 
+									// 		the metadata feature upon request...
+									ImageGridMetadata: 'partial', 
+								})
 
-			} else if(logger === true){
-				force = true
-				logger = arguments[2] }
+						// read the metadata...
+						var exif = metadata.exif 
+							&& exifReader(metadata.exif) 
+						exif
+							&& Object.assign(
+								img.metadata, 
+								exifReader2exiftool(exif, metadata.xmp))
 
-			// NOTE: we are caching this to avoid messing things up when 
-			// 		loading before this was finished...
-			var cached_images = this.images
+						// if image too large, generate preview(s)...
+						// XXX EXPERIMENTAL...
+						var size_threshold = that.config['preview-generate-threshold']
+						if(size_threshold
+								&& img.preview == null
+								&& Math.max(metadata.width, metadata.height) > size_threshold){
+							logger && logger.emit('Image too large', gid)
+							// XXX make this more generic...
+							// 		...if 'loadImages' should create previews in tmp...
+							that.location.load == 'loadIndex'
+								&& that.makePreviews(gid, 
+									that.config['preview-sizes-priority'] || 1080,
+									base_path,
+									logger) }
 
-			// get/normalize images...
-			//images = images || this.current
-			images = images 
-				|| 'current'
-			// keywords...
-			images = 
-				images == 'all' ? 
-					this.data.getImages('all')
-				: images == 'loaded' ?
-					(this.ribbons ?
-						this.ribbons.getImageGIDs()
-						: this.data.getImages('all'))
-				: images == 'current' ? 
-					this.current
-				: images
-			images = (images instanceof Array ? 
-					images 
-					: [images])
-				.filter(function(gid){
-					return !that.__cache_metadata_reading
-						|| !that.__cache_metadata_reading.has(gid) })
+						that.markChanged
+							&& that.markChanged('images', [gid])
+						that.ribbons
+							&& that.ribbons.updateImage(gid) 
 
-			logger = logger !== false ?
-				(logger || this.logger)
-				: false
-			logger = logger 
-				&& logger.push('Caching image metadata', {onclose: abort, quiet: true})
-			logger && logger.emit('queued', images)
-
-			/*/ XXX set this to tmp for .location.load =='loadImages'
-			// XXX add preview cache directory...
-			// 		- user defined path
-			// 		- cleanable 
-			// 			partially (remove orphans) / full...
-			// 		- not sure how to index...
-			var base_path = that.location.load == 'loadIndex' ?
-				null
-				: tmp
-			/*/
-			var base_path
-			//*/
-
-			return images
-				.mapChunks(CHUNK_SIZE, function(gid){
-					// abort...
-					if(STOP){
-						throw Array.STOP('aborted') }
-
-					var img = cached_images[gid]
-					var path = img && that.getImagePath(gid)
-					;(that.__cache_metadata_reading = 
-							that.__cache_metadata_reading || new Set())
-						.add(gid)
-
-					// skip...
-					if(!(img && path
-							&& (force
-								// high priority must be preset...
-								|| (img.orientation == null
-									&& img.flipped == null)
-								// update metadata...
-								|| (img.metadata || {}).ImageGridMetadata == null))){
-						skipping(gid)
-						return }
-
-					return sharp(that.getImagePath(gid))
-						.metadata()
-						.catch(function(){
-							skipping(gid) })
-						.then(function(metadata){
-							// no metadata...
-							if(metadata == null){
-								skipping(gid)
-								return }
-
-							var o = normalizeOrientation(metadata.orientation)
-							;(force || img.orientation == null)
-								// NOTE: we need to set orientation to something
-								// 		or we'll check it again and again...
-								&& (img.orientation = o.orientation || 0)
-							;(force || img.flipped == null)
-								&& (img.flipped = o.flipped)
-
-							// mark metadata as partially read...
-							// NOTE: this will intentionally overwrite the 
-							// 		previous reader mark/mode...
-							img.metadata =
-								Object.assign(
-									img.metadata || {}, 
-									{ 
-										ImageGridMetadataReader: 'sharp/exif-reader/ImageGrid',
-										// mark metadata as partial read...
-										// NOTE: partial metadata will get reread by 
-										// 		the metadata feature upon request...
-										ImageGridMetadata: 'partial', 
-									})
-
-							// read the metadata...
-							var exif = metadata.exif 
-								&& exifReader(metadata.exif) 
-							exif
-								&& Object.assign(
-									img.metadata, 
-									exifReader2exiftool(exif, metadata.xmp))
-
-							// if image too large, generate preview(s)...
-							// XXX EXPERIMENTAL...
-							var size_threshold = that.config['preview-generate-threshold']
-							if(size_threshold
-									&& img.preview == null
-									&& Math.max(metadata.width, metadata.height) > size_threshold){
-								logger && logger.emit('Image too large', gid)
-								// XXX make this more generic...
-								// 		...if 'loadImages' should create previews in tmp...
-								that.location.load == 'loadIndex'
-									&& that.makePreviews(gid, 
-										that.config['preview-sizes-priority'] || 1080,
-										base_path,
-										logger) }
-
-							that.markChanged
-								&& that.markChanged('images', [gid])
-							that.ribbons
-								&& that.ribbons.updateImage(gid) 
-
-							return done(gid) }) }) 
-				.then(function(res){
-					ticket.resolve(res)
-					// XXX do we need this???
-					return res == 'aborted' ?
-						Promise.reject('aborted')
-						: res 
-				}) })],
-	cacheAllMetadata: ['- Sharp|Image/',
-		core.doc`Cache all metadata
-		NOTE: this is a shorthand to .cacheMetadata('all', ..)`,
+						return gid }) })],
+	cacheAllMetadata: ['- Sharp/Image/',
 		'cacheMetadata: "all" ...'],
 
-
 	// shorthands...
 	// XXX do we need these???
 	// 		...better have a task manager UI...
@@ -889,10 +795,10 @@ module.Sharp = core.ImageGridFeatures.Feature({
 				// 		drawn...
 				;((this.images[gid] || {}).metadata || {}).ImageGridMetadata
 					|| this.cacheMetadata('sync', gid, false) 
-						.then(function([res]){
+						.then(function(res){
 							res 
 								&& that.logger 
-									&& that.logger.emit('Cached metadata for', gid) }) }],
+								&& that.logger.emit('Cached metadata for', gid) }) }],
 
 		// XXX need to:
 		// 		- if image too large to set the preview to "loading..."
diff --git a/Viewer/package-lock.json b/Viewer/package-lock.json
index d08393b4..2c666d45 100755
--- a/Viewer/package-lock.json
+++ b/Viewer/package-lock.json
@@ -1110,9 +1110,9 @@
       "integrity": "sha512-9kZM80Js9/eTwXN9VXwLDC1wDJ7gIAdYU9GIzb5KJmNcLAMaW+zhgFrwFFMrcSfggUuadgnqSrS41E4XLe8JZw=="
     },
     "ig-types": {
-      "version": "5.0.30",
-      "resolved": "https://registry.npmjs.org/ig-types/-/ig-types-5.0.30.tgz",
-      "integrity": "sha512-qiE99PB96iEYeygRQTaB1CnJR+1AAAQ/qMmXBKVsbIroAQPveRVXptJIEcPGu6t8AdshibcaLkmLvcEHjMbN0A==",
+      "version": "5.0.32",
+      "resolved": "https://registry.npmjs.org/ig-types/-/ig-types-5.0.32.tgz",
+      "integrity": "sha512-AKfatN0z3hURn9J7JSCaImZnkpr42WMozVLBJeAeC0urkLEU3NUcMEmuDR57dsI5vu9A3d+tyIiRxd5If/3VaQ==",
       "requires": {
         "ig-object": "^5.4.12",
         "object-run": "^1.0.1"
diff --git a/Viewer/package.json b/Viewer/package.json
index b68af5f5..dc81e786 100755
--- a/Viewer/package.json
+++ b/Viewer/package.json
@@ -32,7 +32,7 @@
     "ig-argv": "^2.15.0",
     "ig-features": "^3.4.2",
     "ig-object": "^5.4.12",
-    "ig-types": "^5.0.30",
+    "ig-types": "^5.0.32",
     "moment": "^2.29.1",
     "object-run": "^1.0.1",
     "requirejs": "^2.3.6",