reworked queued action + now .cacheMetadata(..) is queued...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>
This commit is contained in:
Alex A. Naanou 2020-12-02 18:49:45 +03:00
parent 71b9b444cb
commit 7122c619f4
6 changed files with 251 additions and 268 deletions

View File

@ -2520,6 +2520,7 @@ function(func){
// ...would also be nice to automate this via a chunk iterator so
// as not to block...
// XXX handle errors... (???)
// XXX revise logging and logger passing...
var queuedAction =
module.queuedAction =
function(name, func){
@ -2543,25 +2544,69 @@ function(name, func){
}) }
//
//
// queueHandler(name[, opts][, arg_handler], func)
// -> action
//
//
// arg_handler(...args)
// -> [items, ...args]
//
//
// action(items, ...args)
// -> promise
//
// action('sync', items, ...args)
// -> promise
//
//
// func(item, ...args)
// -> res
//
//
// XXX should 'sync' set .sync_start or just .runTask(..)
// XXX check if item is already in queue...
var queueHandler =
module.queueHandler =
function(name, func){
var args = [...arguments]
func = args.pop()
var arg_handler =
typeof(args.last()) == 'function'
&& args.pop()
var [name, opts] = args
return object.mixin(
Queued(function(items, ...args){
var that = this
return new Promise(function(resolve, reject){
var q = that.queue(name,
Object.assign(
{},
opts || {},
{ handler: function(item){
// sync start...
if(arguments[0] == 'sync'){
var [sync, items, ...args] = arguments }
// prep queue...
var q = that.queue(name,
Object.assign(
{},
opts || {},
{ handler: function(item){
return func.call(that, item, ...args) } }))
q.push(...(items instanceof Array ? items : [items]))
q.then(resolve, reject) }) }),
sync
&& (q.sync_start = true)
// pre-process args...
arg_handler
&& ([items, ...args] =
arg_handler.call(this, q, items, ...args))
// fill the queue...
q.push(...(items instanceof Array ? items : [items]))
// make a promise...
var res = new Promise(function(resolve, reject){
q.then(resolve, reject) })
// sync start...
// NOTE: we need to explicitly .start() here because the
// queue could be waiting for a timeout
sync
&& q.start()
return res }),
{
toString: function(){
return `core.queueHandler('${name}',\n\t${
@ -2653,13 +2698,13 @@ var TaskActions = actions.Actions({
get queues(){
return (this.__queues = this.__queues || {}) },
// XXX revise signature...
// XXX revise logging and logger passing...
// XXX need better error flow...
queue: doc('Get or create a queue task',
doc`Get or create a queue task...
.queue(name)
.queue(name[, options][, logger])
.queue(name, options)
-> queue
If a queue with the given name already exits it will be returned
@ -2675,7 +2720,7 @@ var TaskActions = actions.Actions({
NOTE: for queue-specific options see ig-types/runner's Queue(..)
`,
function(name, options, logger){
function(name, options){
var that = this
var queue = this.queues[name]
@ -2692,21 +2737,25 @@ var TaskActions = actions.Actions({
// && logger.emit('close')
delete that.queues[name] } }
var logger = logger || this.logger
options = options || {}
var logger = options.logger || this.logger
//logger = logger && logger.push(name)
logger = logger
&& logger.push(name, {onclose: abort, quiet: !!options.quiet})
logger
&& (options.logger = logger)
queue = this.queues[name] =
runner.Queue(options || {})
// setup logging...
if(logger){
queue
.on('tasksAdded', function(evt, t){ logger.emit('added', t) })
.on('taskCompleted', function(evt, t){ logger.emit('done', t) })
.on('taskFailed', function(evt, t, err){ logger.emit('skipped', t, err) })
queue.logger = logger }
queue
.on('tasksAdded', function(evt, t){
this.logger && this.logger.emit('added', t) })
.on('taskCompleted', function(evt, t){
this.logger && this.logger.emit('done', t) })
.on('taskFailed', function(evt, t, err){
this.logger && this.logger.emit('skipped', t, err) })
// cleanup...
queue
.then(

View File

@ -280,7 +280,25 @@ var ExampleActions = actions.Actions({
this.exampleQueuedAction(timeout) } }],
exampleQueueHandlerAction: ['- Test/',
core.queueHandler('exampleQueueHandlerAction', {quiet: true},
core.queueHandler('exampleQueueHandlerAction',
{quiet: true},
function(item, ...args){
console.log('Queue handler action!!', item, ...args)
return new Promise(function(resolve){
setTimeout(resolve, 100) }) })],
exampleQueueHandlerActionWArgs: ['- Test/',
core.queueHandler('exampleQueueHandlerActionWArgs',
{quiet: true},
function(queue, from=0, to=100, ...args){
var items = []
var reverse = from > to
reverse
&& ([from, to] = [to+1, from+1])
for(var i=from; i<to; i++){
items.push(i) }
reverse
&& items.reverse()
return [items, ...args] },
function(item, ...args){
console.log('Queue handler action!!', item, ...args)
return new Promise(function(resolve){

View File

@ -115,69 +115,79 @@ var MetadataReaderActions = actions.Actions({
NOTE: also see: .cacheMetadata(..)
`,
core.queueHandler('Read image metadata', function(image, force){
var that = this
core.queueHandler('Read image metadata',
function(image, force){
return [
images == 'all' ?
this.images.keys()
: images == 'loaded' ?
this.data.getImages('loaded')
: images,
force,
] },
function(image, force){
var that = this
var gid = this.data.getImage(image)
var img = this.images && this.images[gid]
var gid = this.data.getImage(image)
var img = this.images && this.images[gid]
if(!image && !img){
return false }
if(!image && !img){
return false }
//var full_path = path.normalize(img.base_path +'/'+ img.path)
var full_path = this.getImagePath(gid)
//var full_path = path.normalize(img.base_path +'/'+ img.path)
var full_path = this.getImagePath(gid)
return new Promise(function(resolve, reject){
if(!force
&& (img.metadata || {}).ImageGridMetadata == 'full'){
return resolve(img.metadata) }
return new Promise(function(resolve, reject){
if(!force
&& (img.metadata || {}).ImageGridMetadata == 'full'){
return resolve(img.metadata) }
fs.readFile(full_path, function(err, file){
if(err){
return reject(err) }
// read stat...
if(!that.images[gid].birthtime){
var img = that.images[gid]
var stat = fs.statSync(full_path)
img.atime = stat.atime
img.mtime = stat.mtime
img.ctime = stat.ctime
img.birthtime = stat.birthtime
img.size = stat.size
}
// read image metadata...
exiftool.metadata(file, function(err, data){
fs.readFile(full_path, function(err, file){
if(err){
reject(err)
return reject(err) }
} else if(data.error){
reject(data)
// read stat...
if(!that.images[gid].birthtime){
var img = that.images[gid]
var stat = fs.statSync(full_path)
} else {
// convert to a real dict...
// NOTE: exiftool appears to return an array
// object rather than an actual dict/object
// and that is not JSON compatible....
that.images[gid].metadata =
Object.assign(
// XXX do we need to update or overwrite??
that.images[gid].metadata || {},
data,
{
ImageGridMetadataReader: 'exiftool/ImageGrid',
// mark metadata as full read...
ImageGridMetadata: 'full',
})
that.markChanged
&& that.markChanged('images', [gid]) }
img.atime = stat.atime
img.mtime = stat.mtime
img.ctime = stat.ctime
img.birthtime = stat.birthtime
resolve(data) }) }) }) })],
img.size = stat.size
}
// read image metadata...
exiftool.metadata(file, function(err, data){
if(err){
reject(err)
} else if(data.error){
reject(data)
} else {
// convert to a real dict...
// NOTE: exiftool appears to return an array
// object rather than an actual dict/object
// and that is not JSON compatible....
that.images[gid].metadata =
Object.assign(
// XXX do we need to update or overwrite??
that.images[gid].metadata || {},
data,
{
ImageGridMetadataReader: 'exiftool/ImageGrid',
// mark metadata as full read...
ImageGridMetadata: 'full',
})
that.markChanged
&& that.markChanged('images', [gid]) }
resolve(data) }) }) }) })],
readAllMetadata: ['File/Read all metadata',
'readMetadata: images.gids ...'],
'readMetadata: "all" ...'],
// XXX take image Metadata and write it to target...
writeMetadata: ['- Image/Set metadata data',

View File

@ -376,7 +376,7 @@ var SharpActions = actions.Actions({
&& Math.max(m.width, m.height) < size)
|| (fit == 'outside'
&& Math.min(m.width, m.height) < size)){
logger && logger.emit('skipping', gid)
//logger && logger.emit('skipping', gid)
return }
// continue...
return img })
@ -396,7 +396,7 @@ var SharpActions = actions.Actions({
fse.removeSync(to)
// skip...
} else {
logger && logger.emit('skipping', gid)
//logger && logger.emit('skipping', gid)
return } }
// write...
@ -588,9 +588,8 @@ var SharpActions = actions.Actions({
Promise.reject('aborted')
: res.flat() }) })],
// XXX will this be better off as a queueHandler(..) ???
// XXX add support for offloading the processing to a thread/worker...
__cache_metadata_reading: null,
// XXX revise logging and logger passing...
cacheMetadata: ['- Sharp|Image/',
core.doc`Cache metadata
@ -638,203 +637,110 @@ var SharpActions = actions.Actions({
NOTE: this will effectively update metadata format to the new spec...
NOTE: for info on full metadata format see: .readMetadata(..)
`,
core.sessionTaskAction('cacheMetadata', function(ticket, images, logger){
var that = this
core.queueHandler('Cache image metadata',
{quiet: true},
// parse args...
function(queue, image, logger){
var force = false
if(image === true){
var [force, image, logger] = arguments }
return [
image == 'all' ?
this.images.keys()
: image == 'loaded' ?
this.data.getImages('loaded')
: (image || 'current'),
force,
// XXX
logger || this.logger,
] },
function(image, force, logger){
var that = this
// setup runtime interactions...
//
// NOTE: we will resolve the ticket when we are fully done
// and not on stop...
var STOP = false
ticket
.onmessage('stop', function(){
STOP = true })
.then(function(){
// close progress bar...
// NOTE: if we have multiple tasks let the last one
// close the progress bar...
if(that.tasks.titled(ticket.title).length == 0){
logger
&& logger.emit('close') }
that.off('clear', on_close)
// cleanup...
delete that.__cache_metadata_reading })
// clear the progress bar for the next session...
var on_close
this.one('clear', on_close = function(){
logger && logger.emit('close') })
// XXX cache the image data???
var gid = this.data.getImage(image)
var img = this.images[gid]
var path = img && that.getImagePath(gid)
// universal task abort...
// NOTE: this will abort all the tasks of this type...
var abort = function(){
that.tasks.stop(ticket.title) }
// XXX
//var base_path = that.location.load == 'loadIndex' ?
// null
// : tmp
//var base_path = img && img.base_path
var base_path
var CHUNK_SIZE = 4
// skip...
if(!(img && path
&& (force
// high priority must be preset...
|| (img.orientation == null
&& img.flipped == null)
// update metadata...
|| (img.metadata || {}).ImageGridMetadata == null))){
return }
// handle logging and processing list...
// NOTE: these will maintain .__cache_metadata_reading helping
// avoid processing an image more than once at the same
// time...
var done = function(gid, msg){
logger && logger.emit(msg || 'done', gid)
if(that.__cache_metadata_reading){
that.__cache_metadata_reading.delete(gid) }
return gid }
var skipping = function(gid){
return done(gid, 'skipping') }
// XXX handle/report errors...
return sharp(that.getImagePath(gid))
.metadata()
.then(function(metadata){
// no metadata...
if(metadata == null){
return }
var force = false
if(images === true){
force = true
images = null
var o = normalizeOrientation(metadata.orientation)
;(force || img.orientation == null)
// NOTE: we need to set orientation to something
// or we'll check it again and again...
&& (img.orientation = o.orientation || 0)
;(force || img.flipped == null)
&& (img.flipped = o.flipped)
} else if(logger === true){
force = true
logger = arguments[2] }
// mark metadata as partially read...
// NOTE: this will intentionally overwrite the
// previous reader mark/mode...
img.metadata =
Object.assign(
img.metadata || {},
{
ImageGridMetadataReader: 'sharp/exif-reader/ImageGrid',
// mark metadata as partial read...
// NOTE: partial metadata will get reread by
// the metadata feature upon request...
ImageGridMetadata: 'partial',
})
// NOTE: we are caching this to avoid messing things up when
// loading before this was finished...
var cached_images = this.images
// read the metadata...
var exif = metadata.exif
&& exifReader(metadata.exif)
exif
&& Object.assign(
img.metadata,
exifReader2exiftool(exif, metadata.xmp))
// get/normalize images...
//images = images || this.current
images = images
|| 'current'
// keywords...
images =
images == 'all' ?
this.data.getImages('all')
: images == 'loaded' ?
(this.ribbons ?
this.ribbons.getImageGIDs()
: this.data.getImages('all'))
: images == 'current' ?
this.current
: images
images = (images instanceof Array ?
images
: [images])
.filter(function(gid){
return !that.__cache_metadata_reading
|| !that.__cache_metadata_reading.has(gid) })
// if image too large, generate preview(s)...
// XXX EXPERIMENTAL...
var size_threshold = that.config['preview-generate-threshold']
if(size_threshold
&& img.preview == null
&& Math.max(metadata.width, metadata.height) > size_threshold){
logger && logger.emit('Image too large', gid)
// XXX make this more generic...
// ...if 'loadImages' should create previews in tmp...
that.location.load == 'loadIndex'
&& that.makePreviews(gid,
that.config['preview-sizes-priority'] || 1080,
base_path,
logger) }
logger = logger !== false ?
(logger || this.logger)
: false
logger = logger
&& logger.push('Caching image metadata', {onclose: abort, quiet: true})
logger && logger.emit('queued', images)
that.markChanged
&& that.markChanged('images', [gid])
that.ribbons
&& that.ribbons.updateImage(gid)
/*/ XXX set this to tmp for .location.load =='loadImages'
// XXX add preview cache directory...
// - user defined path
// - cleanable
// partially (remove orphans) / full...
// - not sure how to index...
var base_path = that.location.load == 'loadIndex' ?
null
: tmp
/*/
var base_path
//*/
return images
.mapChunks(CHUNK_SIZE, function(gid){
// abort...
if(STOP){
throw Array.STOP('aborted') }
var img = cached_images[gid]
var path = img && that.getImagePath(gid)
;(that.__cache_metadata_reading =
that.__cache_metadata_reading || new Set())
.add(gid)
// skip...
if(!(img && path
&& (force
// high priority must be preset...
|| (img.orientation == null
&& img.flipped == null)
// update metadata...
|| (img.metadata || {}).ImageGridMetadata == null))){
skipping(gid)
return }
return sharp(that.getImagePath(gid))
.metadata()
.catch(function(){
skipping(gid) })
.then(function(metadata){
// no metadata...
if(metadata == null){
skipping(gid)
return }
var o = normalizeOrientation(metadata.orientation)
;(force || img.orientation == null)
// NOTE: we need to set orientation to something
// or we'll check it again and again...
&& (img.orientation = o.orientation || 0)
;(force || img.flipped == null)
&& (img.flipped = o.flipped)
// mark metadata as partially read...
// NOTE: this will intentionally overwrite the
// previous reader mark/mode...
img.metadata =
Object.assign(
img.metadata || {},
{
ImageGridMetadataReader: 'sharp/exif-reader/ImageGrid',
// mark metadata as partial read...
// NOTE: partial metadata will get reread by
// the metadata feature upon request...
ImageGridMetadata: 'partial',
})
// read the metadata...
var exif = metadata.exif
&& exifReader(metadata.exif)
exif
&& Object.assign(
img.metadata,
exifReader2exiftool(exif, metadata.xmp))
// if image too large, generate preview(s)...
// XXX EXPERIMENTAL...
var size_threshold = that.config['preview-generate-threshold']
if(size_threshold
&& img.preview == null
&& Math.max(metadata.width, metadata.height) > size_threshold){
logger && logger.emit('Image too large', gid)
// XXX make this more generic...
// ...if 'loadImages' should create previews in tmp...
that.location.load == 'loadIndex'
&& that.makePreviews(gid,
that.config['preview-sizes-priority'] || 1080,
base_path,
logger) }
that.markChanged
&& that.markChanged('images', [gid])
that.ribbons
&& that.ribbons.updateImage(gid)
return done(gid) }) })
.then(function(res){
ticket.resolve(res)
// XXX do we need this???
return res == 'aborted' ?
Promise.reject('aborted')
: res
}) })],
cacheAllMetadata: ['- Sharp|Image/',
core.doc`Cache all metadata
NOTE: this is a shorthand to .cacheMetadata('all', ..)`,
return gid }) })],
cacheAllMetadata: ['- Sharp/Image/',
'cacheMetadata: "all" ...'],
// shorthands...
// XXX do we need these???
// ...better have a task manager UI...
@ -889,10 +795,10 @@ module.Sharp = core.ImageGridFeatures.Feature({
// drawn...
;((this.images[gid] || {}).metadata || {}).ImageGridMetadata
|| this.cacheMetadata('sync', gid, false)
.then(function([res]){
.then(function(res){
res
&& that.logger
&& that.logger.emit('Cached metadata for', gid) }) }],
&& that.logger.emit('Cached metadata for', gid) }) }],
// XXX need to:
// - if image too large to set the preview to "loading..."

View File

@ -1110,9 +1110,9 @@
"integrity": "sha512-9kZM80Js9/eTwXN9VXwLDC1wDJ7gIAdYU9GIzb5KJmNcLAMaW+zhgFrwFFMrcSfggUuadgnqSrS41E4XLe8JZw=="
},
"ig-types": {
"version": "5.0.30",
"resolved": "https://registry.npmjs.org/ig-types/-/ig-types-5.0.30.tgz",
"integrity": "sha512-qiE99PB96iEYeygRQTaB1CnJR+1AAAQ/qMmXBKVsbIroAQPveRVXptJIEcPGu6t8AdshibcaLkmLvcEHjMbN0A==",
"version": "5.0.32",
"resolved": "https://registry.npmjs.org/ig-types/-/ig-types-5.0.32.tgz",
"integrity": "sha512-AKfatN0z3hURn9J7JSCaImZnkpr42WMozVLBJeAeC0urkLEU3NUcMEmuDR57dsI5vu9A3d+tyIiRxd5If/3VaQ==",
"requires": {
"ig-object": "^5.4.12",
"object-run": "^1.0.1"

View File

@ -32,7 +32,7 @@
"ig-argv": "^2.15.0",
"ig-features": "^3.4.2",
"ig-object": "^5.4.12",
"ig-types": "^5.0.30",
"ig-types": "^5.0.32",
"moment": "^2.29.1",
"object-run": "^1.0.1",
"requirejs": "^2.3.6",