From 701a26919fb4960c8186aebe68d83e919a2c80b8 Mon Sep 17 00:00:00 2001 From: "Alex A. Naanou" Date: Tue, 15 Dec 2020 05:36:37 +0300 Subject: [PATCH] nested queues... Signed-off-by: Alex A. Naanou --- Viewer/e.js | 1 + Viewer/features/core.js | 84 ++++---- Viewer/features/examples.js | 13 ++ Viewer/features/sharp.js | 394 ++++++++++++++++++++++++++++++++++++ Viewer/ig.js | 5 +- 5 files changed, 457 insertions(+), 40 deletions(-) diff --git a/Viewer/e.js b/Viewer/e.js index c0d05f99..8e956315 100644 --- a/Viewer/e.js +++ b/Viewer/e.js @@ -104,6 +104,7 @@ function createWindow(){ WIN = new BrowserWindow({ webPreferences: { nodeIntegration: true, + nodeIntegrationInWorker: true, contextIsolation: false, enableRemoteModule: true, }, diff --git a/Viewer/features/core.js b/Viewer/features/core.js index 383d4161..1f29993a 100755 --- a/Viewer/features/core.js +++ b/Viewer/features/core.js @@ -2695,56 +2695,66 @@ function(title, func){ return object.mixin( action = Queued(function(items, ...args){ var that = this + // sync start... if(arguments[0] == 'sync' || arguments[0] == 'async'){ var [sync, items, ...args] = arguments } + var q + var inputs = [items, ...args] + + // Define the runner and prepare... + // // sync mode -- run action outside of queue... // NOTE: running the queue in sync mode is not practical as // the results may depend on queue configuration and // size... if(sync == 'sync'){ - // pre-process args... - arg_handler - && ([items, ...args] = - arg_handler.call(this, undefined, items, ...args)) - // run... - return Promise.all( - (items instanceof Array ? - items - : [items]) - .map(function(item){ - return func.call(that, item, ...args) })) + var run = function([items, ...args]){ + return Promise.all( + (items instanceof Array ? + items + : [items]) + .map(function(item){ + return func.call(that, item, ...args) })) } // queue mode... } else { // prep queue... - var q = that.queue(title, - Object.assign( - {}, - opts || {}, - { - // XXX not sure about this... - //auto_stop: true, - handler: function([item, args]){ - return func.call(that, item, ...(args || [])) }, - })) - q.title = action.name - // pre-process args... - arg_handler - && ([items, ...args] = - arg_handler.call(this, q, items, ...args)) - // fill the queue... - // NOTE: we are also adding a ref to args here to keep things consistent... - args.length > 0 - && (args = [args]) - q.push(...(items instanceof Array ? - items.map(function(e){ - return [e, ...args] }) - : [items, ...args])) - // make a promise... - return new Promise(function(resolve, reject){ - q.then(resolve, reject) }) } }), + q = that.queue(title, + Object.assign( + {}, + opts || {}, + { + // XXX not sure about this... + //auto_stop: true, + handler: function([item, args]){ + return func.call(that, item, ...(args || [])) }, + })) + q.title = action.name + + var run = function([items, ...args]){ + // fill the queue... + // NOTE: we are also adding a ref to args here to keep things consistent... + args.length > 0 + && (args = [args]) + q.push(...(items instanceof Array ? + items.map(function(e){ + return [e, ...args] }) + : [items, ...args])) + // make a promise... + return new Promise(function(resolve, reject){ + q.then(resolve, reject) }) } } + + // pre-process args... + arg_handler + && (inputs = arg_handler.call(this, q, inputs[0], ...inputs.slice(1))) + // run... + return (inputs instanceof Promise + || inputs instanceof runner.Queue) ? + inputs.then(function(items){ + return run([items, ...args]) }) + : run(inputs) }), { title, toString: function(){ diff --git a/Viewer/features/examples.js b/Viewer/features/examples.js index 354c1f26..269c5575 100755 --- a/Viewer/features/examples.js +++ b/Viewer/features/examples.js @@ -341,6 +341,19 @@ var ExampleActions = actions.Actions({ return new Promise(function(resolve){ setTimeout(resolve, timeout || 100) }) })], + exampleChainedQueueHandler: ['- Test/', + core.queueHandler('Main queue', + core.queueHandler('Sub queue', + function(outer_queue, inner_queue, items, ...args){ + console.log('### PRE-PREP', items, ...args) + return [items, ...args] }, + function(item, ...args){ + console.log('### PREP', item, ...args) + return item+1 }), + function(item, ...args){ + console.log('### HANDLE', item, ...args) + return item*2 }) ], + }) var Example = diff --git a/Viewer/features/sharp.js b/Viewer/features/sharp.js index e01e9c42..a0407c5e 100755 --- a/Viewer/features/sharp.js +++ b/Viewer/features/sharp.js @@ -393,6 +393,400 @@ var SharpActions = actions.Actions({ // XXX what should we return??? return to }) }) }) })], + _makeResizedImage: ['- Image/', + core.doc`Make resized image(s)... + + .makeResizedImage(gid, size, path[, options]) + .makeResizedImage(gids, size, path[, options]) + -> promise + + + Image size formats: + 500px - resize to make image's *largest* dimension 500 pixels (default). + 500p - resize to make image's *smallest* dimension 500 pixels. + 500 - same as 500px + + + options format: + { + // output image name / name pattern... + // + // NOTE: for multiple images this should be a pattern and not an + // explicit name... + // NOTE: if not given this defaults to: "%n" + name: null | , + + // image name pattern data... + // + // NOTE: for more info on pattern see: .formatImageName(..) + data: null | { .. }, + + // if true and image is smaller than size enlarge it... + // + // default: null / false + enlarge: null | true, + + // overwrite, backup or skip (default) existing images... + // + // default: null / false + overwrite: null | true | 'backup', + + // if true do not write an image if it's smaller than size... + // + // default: null / false + skipSmaller: null | true, + + // XXX not implemented... + transform: ..., + crop: ..., + + timestamp: ..., + logger: ..., +, } + + + NOTE: all options are optional. + NOTE: this will not overwrite existing images. + `, + core.queueHandler('Make resized image', + // queue the image data... + // NOTE: after this runs we should be completely independent + // of the current index... + // XXX for a very large number of images this can block for + // a substantial amount of time... + function(queue, images, size, path, options){ + var that = this + // sanity check... + if(arguments.length < 4){ + throw new Error('.makeResizedImage(..): ' + +'need at least: images, size and path.') } + // options... + var { + name, + data, + } = options || {} + name = name || '%n' + // [source, to, image], ...args + return [ + ((images == null || images == 'all') ? + this.data.getimages('all') + : images == 'current' ? + [this.current] + : images instanceof array ? + images + : [images]) + .map(function(gid){ + var image = that.images[gid] + // skip non-images... + if(!image || !['image', null, undefined] + .includes(image.type)){ + return [] } + return [[ + // source... + that.getimagepath(gid), + // target... + pathlib.resolve( + that.location.path, + pathlib.join( + path, + // if name is not a pattern do not re-format it... + name.includes('%') ? + that.formatimagename(name, gid, data || {}) + : name)), + // image data... + // note: we include only the stuff we need... + { + orientation: image.orientation, + flipped: image.flipped, + // crop... + }, + ]] }) + .flat(), + ...[...arguments].slice(2), + ]}, + function([source, to, image], size, _, options={}){ + // sizing... + var fit = + typeof(size) == typeof('str') ? + (size.endsWith('px') ? + 'inside' + : size.endsWith('p') ? + 'outside' + : 'inside') + : 'inside' + size = parseInt(size) + // options... + var { + enlarge, + skipSmaller, + overwrite, + transform, + timestamp, + backupImagePattern, + //logger, + } = options + // defaults... + transform = transform === undefined ? + true + : transform + timestamp = timestamp || Date.timeStamp() + // backup by default... + overwrite = overwrite === undefined ? + 'backup' + : overwrite + backupImagePattern = + (backupImagePattern + || '${PATH}.${TIMESTAMP}${COUNT}.bak') + .replace(/\${PATH}|$PATH/, to) + .replace(/\${TIMESTAMP}|$TIMESTAMP/, timestamp) + // backup... + // NOTE: we are doing the check at the very last moment and + // not here to avoid race conditions as much as practical... + var backupName = function(){ + var i = 0 + var n + do{ + n = backupImagePattern + .replace(/\${COUNT}|$COUNT/, i++ ? '.'+i : i) + } while(fse.existsSync(n)) + return n } + + var img = sharp(source) + return (skipSmaller ? + // skip if smaller than size... + img + .metadata() + .then(function(m){ + // skip... + if((fit == 'inside' + && Math.max(m.width, m.height) < size) + || (fit == 'outside' + && Math.min(m.width, m.height) < size)){ + return } + // continue... + return img }) + : Promise.resolve(img)) + // prepare to write... + .then(function(img){ + return img + && ensureDir(pathlib.dirname(to)) + .then(function(){ + // handle existing image... + if(fse.existsSync(to)){ + // rename... + if(overwrite == 'backup'){ + fse.renameSync(to, backupName(to)) + // remove... + } else if(overwrite){ + fse.removeSync(to) + // skip... + } else { + return Promise.reject('target exists') } } + // write... + return img + .clone() + // handle transform (.orientation / .flip) and .crop... + .run(function(){ + if(transform && (image.orientation || image.flipped)){ + image.orientation + && this.rotate(image.orientation) + image.flipped + && image.flipped.includes('horizontal') + && this.flip() } + image.flipped + && image.flipped.includes('vertical') + && this.flop() + // XXX CROP + //if(crop){ + // // XXX + //} + }) + .resize({ + width: size, + height: size, + fit: fit, + withoutEnlargement: !enlarge, + }) + .withMetadata() + .toFile(to) + .then(function(){ + // XXX what should we return??? + return to }) }) }) })], + + // XXX we need to split this into two stages: + // - session queue handler + // - global queue handler + // i.e. call the second queue generator when the first one completes... + // or in other works chain queues -- essentially this is like + // calling .then(..) on a queue but doing it at definition... + makeResizedImage2: ['- Image/', + core.doc` + `, + core.queueHandler('Making resized image', + // prepare the data for image resizing (session queue)... + core.sessionQueueHandler('Gathering image data for resizing', + // prepare the input index-dependant data in a fast way... + function(inner_queue, outer_queue, images, size, path, options){ + // sanity check... + if(arguments.length < 4){ + throw new Error('.makeResizedImage(..): ' + +'need at least: images, size and path.') } + return [ + (images == null || images == 'all') ? + this.data.getImages('all') + : images == 'current' ? + [this.current] + : images instanceof Array ? + images + : [images], + ...[...arguments].slice(3), + ]}, + // prepare index independent data, this can be a tad slow... + function(gid, size, path, options={}){ + var image = this.images[gid] + // options... + var { + name, + data, + } = options || {} + name = name || '%n' + // skip non-images... + if(!image || !['image', null, undefined] + .includes(image.type)){ + return [] } + return [ + // source... + this.getImagePath(gid), + // target... + pathlib.resolve( + this.location.path, + pathlib.join( + path, + // if name is not a pattern do not re-format it... + name.includes('%') ? + this.formatImageName(name, gid, data || {}) + : name)), + // image data... + // note: we include only the stuff we need... + { + orientation: image.orientation, + flipped: image.flipped, + // crop... + }, + ] }), + // do the actual resizing (global queue)... + function([source, to, image], size, _, options={}){ + // XXX handle skipped items -- source, to and image are undefined... + // XXX + + // sizing... + var fit = + typeof(size) == typeof('str') ? + (size.endsWith('px') ? + 'inside' + : size.endsWith('p') ? + 'outside' + : 'inside') + : 'inside' + size = parseInt(size) + // options... + var { + enlarge, + skipSmaller, + overwrite, + transform, + timestamp, + backupImagePattern, + //logger, + } = options + // defaults... + transform = transform === undefined ? + true + : transform + timestamp = timestamp || Date.timeStamp() + // backup by default... + overwrite = overwrite === undefined ? + 'backup' + : overwrite + backupImagePattern = + (backupImagePattern + || '${PATH}.${TIMESTAMP}${COUNT}.bak') + .replace(/\${PATH}|$PATH/, to) + .replace(/\${TIMESTAMP}|$TIMESTAMP/, timestamp) + // backup... + // NOTE: we are doing the check at the very last moment and + // not here to avoid race conditions as much as practical... + var backupName = function(){ + var i = 0 + var n + do{ + n = backupImagePattern + .replace(/\${COUNT}|$COUNT/, i++ ? '.'+i : i) + } while(fse.existsSync(n)) + return n } + + var img = sharp(source) + return (skipSmaller ? + // skip if smaller than size... + img + .metadata() + .then(function(m){ + // skip... + if((fit == 'inside' + && Math.max(m.width, m.height) < size) + || (fit == 'outside' + && Math.min(m.width, m.height) < size)){ + return } + // continue... + return img }) + : Promise.resolve(img)) + // prepare to write... + .then(function(img){ + return img + && ensureDir(pathlib.dirname(to)) + .then(function(){ + // handle existing image... + if(fse.existsSync(to)){ + // rename... + if(overwrite == 'backup'){ + fse.renameSync(to, backupName(to)) + // remove... + } else if(overwrite){ + fse.removeSync(to) + // skip... + } else { + return Promise.reject('target exists') } } + // write... + return img + .clone() + // handle transform (.orientation / .flip) and .crop... + .run(function(){ + if(transform && (image.orientation || image.flipped)){ + image.orientation + && this.rotate(image.orientation) + image.flipped + && image.flipped.includes('horizontal') + && this.flip() } + image.flipped + && image.flipped.includes('vertical') + && this.flop() + // XXX CROP + //if(crop){ + // // XXX + //} + }) + .resize({ + width: size, + height: size, + fit: fit, + withoutEnlargement: !enlarge, + }) + .withMetadata() + .toFile(to) + .then(function(){ + // XXX what should we return??? + return to }) }) }) })], + // XXX this does not update image.base_path -- is this correct??? // XXX add support for offloading the processing to a thread/worker... makePreviews: ['Sharp|File/Make image $previews', diff --git a/Viewer/ig.js b/Viewer/ig.js index 404c07eb..496421c3 100755 --- a/Viewer/ig.js +++ b/Viewer/ig.js @@ -21,15 +21,14 @@ global.scopeDiff = function(cur=global, base=__global){ /*********************************************************************/ require('v8-compile-cache') -// NOTE: this fixes several issues with lib/util conflicting with stuff... +// NOTE: importing this before require fixes several issues with lib/util +// conflicting with stuff... require('repl') - // setup module loaders... require = require('./cfg/requirejs')(require).requirejs require.main = {filename: (nodeRequire.main || {}).filename} - var core = require('features/core') // XXX for some reason if this is not loaded here things break in CLI... // ...setting priority does not help...