From 6b8f9f7c4f49e39316d1466e5d20cac411daa374 Mon Sep 17 00:00:00 2001 From: "Alex A. Naanou" Date: Thu, 9 Jul 2020 02:19:43 +0300 Subject: [PATCH] reqorked logging and progress indication... Signed-off-by: Alex A. Naanou --- ui (gen4)/features/filesystem.js | 134 ++++++++++++++++++++---------- ui (gen4)/features/ui-progress.js | 114 ++++++++++++++++--------- ui (gen4)/imagegrid/file.js | 6 +- ui (gen4)/lib/util.js | 57 ++++++++----- 4 files changed, 206 insertions(+), 105 deletions(-) diff --git a/ui (gen4)/features/filesystem.js b/ui (gen4)/features/filesystem.js index b38fe0e4..185aaca3 100755 --- a/ui (gen4)/features/filesystem.js +++ b/ui (gen4)/features/filesystem.js @@ -491,6 +491,7 @@ var FileSystemLoaderActions = actions.Actions({ // // Returns: Images object // + // XXX revise logging... getImagesInPath: ['- File/', function(path, read_stat, skip_preview_search, logger){ if(path == null){ @@ -503,24 +504,42 @@ var FileSystemLoaderActions = actions.Actions({ this.config['image-file-skip-previews'] : skip_preview_search - // XXX get a logger... logger = logger || this.logger - //logger = logger && logger.push('getImagesInPath') var that = this path = util.normalizePath(path) + // progress... + // XXX this does not appear to run while glob(..) is running... + var found = [] + var update_interval + if(logger){ + that.showProgress + && that.showProgress(logger.path) + update_interval = setInterval(function(){ + found.length > 0 + && logger.emit('found', found) + && (found = []) }, 150) } + // get the image list... return new Promise(function(resolve, reject){ glob(path + '/'+ that.config['image-file-pattern'], { stat: !!read_stat, strict: false, }) + .on('match', function(e){ found.push(e) }) .on('error', function(err){ + update_interval + && clearInterval(update_interval) console.error(err) - reject(err) - }) + reject(err) }) .on('end', function(lst){ + update_interval + && clearInterval(update_interval) + logger && found.length > 0 + && logger.emit('found', found) + && (found = []) + // XXX might be a good idea to make image paths relative to path... //lst = lst.map(function(p){ return pathlib.relative(base, p) }) // XXX do we need to normalize paths after we get them from glob?? @@ -543,8 +562,7 @@ var FileSystemLoaderActions = actions.Actions({ img.size = stat.size // XXX do we need anything else??? - }) - } + }) } // pass on the result... resolve(imgs) @@ -558,9 +576,7 @@ var FileSystemLoaderActions = actions.Actions({ return !skip_preview_search ? //that.getPreviews('all', path, imgs) that.getPreviews('all', index_path, imgs) - : imgs - }) - }], + : imgs }) }], // Load images... // @@ -593,6 +609,7 @@ var FileSystemLoaderActions = actions.Actions({ logger) // load the data... .then(function(imgs){ + logger && logger.emit('loaded', imgs.keys()) return that.loadOrRecover({ images: imgs, data: data.Data.fromArray(imgs.keys()), @@ -604,10 +621,7 @@ var FileSystemLoaderActions = actions.Actions({ } }) .then(function(){ - that.markChanged('none') - }) - }) - }], + that.markChanged('none') }) }) }], // Load images to new ribbon... // @@ -650,6 +664,8 @@ var FileSystemLoaderActions = actions.Actions({ logger) // load the data... .then(function(imgs){ + logger && logger.emit('loaded', imgs.keys()) + that.clearLoaction() var d = that.data @@ -699,19 +715,19 @@ var FileSystemLoaderActions = actions.Actions({ return ['loadIndex', 'loadImages'].includes(this.location.load) || 'disabled' }, }, function(path, logger){ + var that = this path = path || this.location.path if(path == null){ return } - var that = this logger = logger || this.logger logger = logger && logger.push('Load new images') path = util.normalizePath(path) // cache the loaded images... - var loaded = this.images.map(function(gid, img){ return img.path }) + var loaded = new Set(this.images.map(function(gid, img){ return img.path })) //var base_pattern = RegExp('^'+path) return this.getImagesInPath( @@ -721,25 +737,47 @@ var FileSystemLoaderActions = actions.Actions({ logger) // load the data... .then(function(imgs){ + var added = [] + var skipped = [] + var progress = function(){ + skipped.length > 0 + && logger.emit('skipped', skipped) + && (skipped = []) + added.length > 0 + && logger.emit('done', added) + && (added = []) } + // remove the images we already have loaded... + var t = Date.now() imgs.forEach(function(gid, img){ + // XXX this does not let the browser update progress... + Date.now() - t > 200 + && (t = Date.now()) + && progress() // NOTE: we do not need to normalize anything as // both the current path and loaded paths // came from the same code... // XXX is this good enough??? // ...might be a good idea to compare absolute // paths... - if(loaded.indexOf(img.path) >= 0){ - delete imgs[gid] - } - }) + if(loaded.has(img.path)){ + delete imgs[gid] + skipped.push(gid) + } else { + added.push(gid) } }) + + // finalize progress... + if(logger){ + skipped.length > 0 + && logger.emit('skipped', skipped) + added.length > 0 + && logger.emit('done', added) } // nothing new... if(imgs.length == 0){ // XXX logger && logger.emit('loaded', []) - return imgs - } + return imgs } // XXX logger && logger.emit('queued', imgs) @@ -863,39 +901,47 @@ var FileSystemLoaderActions = actions.Actions({ logger = logger && logger.push('Check missing') logger - && this.images - .forEach(function(gid){ - logger.emit('queued', gid)}) + && logger.emit('queued', this.images.keys()) var chunk_size = '100C' + var removed = [] return this.images .map(function(gid, image){ return [gid, image] }) - .mapChunks(chunk_size, function([gid, image]){ - var updated = false + .mapChunks(chunk_size, [ + function([gid, image]){ + var updated = false - image.path - && !fse.existsSync(image.base_path +'/'+ image.path) - && (updated = true) - && logger && rem_logger.emit('queued', gid) + image.path + && !fse.existsSync(image.base_path +'/'+ image.path) + && (updated = true) + && logger + && removed.push(gid) - logger && logger.emit('done', gid) - - return updated ? gid : [] - }) + return updated ? gid : [] }, + // do the logging per chunk... + function(chunk, res){ + logger + && logger.emit('done', chunk.map(function([gid]){ return gid })) + && rem_logger.emit('queued', removed) + && (removed = []) }]) .then(function(res){ res = res.flat() - if(res.length > 0){ - logger && rem_logger.emit('queued', 'data cleanup') - // clear images... - res.forEach(function(gid){ - delete that.images[gid] - logger && rem_logger.emit('done', gid) }) - // clear data... - that.data.clear(res) - logger && rem_logger.emit('done', 'data cleanup') } - return res }) }], + return res.length > 0 ? + res + .mapChunks(chunk_size, [ + // clear images... + function(gid){ + delete that.images[gid] }, + // log... + function(chunk){ + logger && rem_logger.emit('done', chunk) }]) + // clear data... + .then(function(){ + that.data.clear(res) + return res }) + : res }) }], // XXX EXPERIMENTAL... diff --git a/ui (gen4)/features/ui-progress.js b/ui (gen4)/features/ui-progress.js index 7c10a247..c6f51d05 100755 --- a/ui (gen4)/features/ui-progress.js +++ b/ui (gen4)/features/ui-progress.js @@ -20,6 +20,8 @@ var ProgressActions = actions.Actions({ config: { 'progress-fade-duration': 200, 'progress-done-delay': 1000, + + 'progress-update-min': 200, }, // Progress bar widget... @@ -41,20 +43,56 @@ var ProgressActions = actions.Actions({ // .showProgress('text', '+1') // .showProgress('text', '+0', '+1') // + // .showProgress(logger) + // // // XXX add message to be shown... // XXX should we report errors and stoppages??? (error state??) // XXX multiple containers... // XXX shorten the nested css class names... // XXX revise styles... + __progress_cache: null, showProgress: ['- Interface/Show progress bar...', function(text, value, max){ - var viewer = this.dom var that = this + var viewer = this.dom var msg = text instanceof Array ? text.slice(1).join(': ') : null text = text instanceof Array ? text[0] : text + // make sure we do not update too often... + if(value != 'close'){ + var cache = (this.__progress_cache = this.__progress_cache || {}) + cache = cache[text] = cache[text] || {} + + var updateValue = function(name, value){ + var v = cache[name] || 0 + return (cache[name] = + value != null ? + (typeof(value) == typeof('str') && /[+-][0-9]+/.test(value) ? + v + parseInt(value) + : parseInt(value)) + : v) } + + value = updateValue('value', value) + max = updateValue('max', max) + + // update not due yet... + if('timeout' in cache){ + cache.update = true + return + + // set next update point and continue... + } else { + delete cache.update + cache.timeout = setTimeout( + function(){ + var cache = that.__progress_cache[text] || {} + delete cache.timeout + cache.update + && that.showProgress(text) }, + this.config['progress-update-min'] || 200) } } + // container... var container = viewer.find('.progress-container') container = container.length == 0 ? @@ -68,8 +106,8 @@ var ProgressActions = actions.Actions({ // close action... if(value == 'close'){ widget.trigger('progressClose') - return - } + return } + widget = widget.length == 0 ? $('
') .addClass('progress-bar') @@ -87,6 +125,10 @@ var ProgressActions = actions.Actions({ .on('progressClose', function(){ widget .fadeOut(that.config['progress-fade-duration'] || 200, function(){ + var cache = (that.__progress_cache || {}) + cache[text].timeout + && clearTimeout(cache[text].timeout) + delete cache[text] $(this).remove() }) }) .appendTo(container) : widget @@ -99,26 +141,11 @@ var ProgressActions = actions.Actions({ var bar = widget.find('progress') var state = widget.find('.progress-details') - // XXX stub??? - // normalize max and value... - max = max != null ? - (typeof(max) == typeof('str') && /[+-][0-9]+/.test(max) ? - parseInt(bar.attr('max') || 0) + parseInt(max) - : parseInt(max)) - : bar.attr('max') - value = value != null ? - (typeof(value) == typeof('str') && /[+-][0-9]+/.test(value) ? - parseInt(bar.attr('value') || 0) + parseInt(value) - : parseInt(value)) - : bar.attr('value') - // format the message... - // XXX should we add a message after this???? msg = msg ? ': '+msg : '' msg = ' '+ msg - //+ (value && value >= (max || 0) ? ' ('+value+' done)' + (value && value >= (max || 0) ? ' (done)' - : value && max && value != max ? ' ('+ value +' of '+ max +')' + : max && value != max ? ' ('+ (value || 0) +' of '+ max +')' : '...') // update widget... @@ -129,48 +156,59 @@ var ProgressActions = actions.Actions({ state.text(msg) // auto-close... - // XXX make this optional... if(value && value >= (max || 0)){ widget.attr('close-timeout', JSON.stringify(setTimeout(function(){ widget.trigger('progressClose') - }, this.config['progress-done-delay'] || 1000))) - } + }, this.config['progress-done-delay'] || 1000))) } // XXX force the browser to render... //bar.hide(0).show(0) - - // XXX what should we return??? (state, self, controller?) }], // handle logger progress... // XXX revise... handleLogItem: ['- System/', function(path, status, ...rest){ - var msg = this.message + var msg = path.join(': ') + var l = (rest.length == 1 && rest[0] instanceof Array) ? + rest[0].length + : rest.length + + // XXX should we move these to a more accessible spot??? + var add = [ + 'added', + 'queued', + 'found', + ] + var done = [ + 'loaded', + 'done', + 'written', + 'index', + ] + var skipped = [ + 'skipping', + 'skipped', + 'removed', + ] // report progress... // XXX HACK -- need meaningful status... - if(status == 'queued' - || status == 'found'){ - this.showProgress(msg || ['Progress', status], '+0', '+'+rest.length) + if(add.includes(status)){ + this.showProgress(msg, '+0', '+'+l) - } else if(status == 'loaded' || status == 'done' || status == 'written' - || status == 'index'){ - this.showProgress(msg || ['Progress', status], '+'+rest.length) + } else if(done.includes(status)){ + this.showProgress(msg, '+'+l) - } else if(status == 'skipping' || status == 'skipped'){ + } else if(skipped.includes(status)){ // XXX if everything is skipped the indicator does not // get hidden... - //this.showProgress(msg || ['Progress', status], '+0', '-1') - this.showProgress(msg || ['Progress', status], '+'+rest.length) + this.showProgress(msg, '+'+l) // XXX STUB... } else if(status == 'error' ){ - this.showProgress(['Error'].concat(msg), '+0', '+'+rest.length) - //console.log(msg ? - // ' '+ msg.join(': ') + ':' - // : '', ...arguments) + this.showProgress(['Error'].concat(msg), '+0', '+'+l) } }], }) diff --git a/ui (gen4)/imagegrid/file.js b/ui (gen4)/imagegrid/file.js index b3b72abd..336f11b6 100755 --- a/ui (gen4)/imagegrid/file.js +++ b/ui (gen4)/imagegrid/file.js @@ -244,7 +244,7 @@ function(list){ var groupByKeyword = module.groupByKeyword = function(list, from_date, logger){ - logger = logger && logger.push('Grouping by keyword') + //logger = logger && logger.push('Grouping by keyword') var index = {} var queued = 0 @@ -312,8 +312,8 @@ function(list, from_date, logger){ // remove the flags... for(var k in index){ - index[k] = index[k].map(function(e){ return e[1] }) - } + index[k] = index[k] + .map(function(e){ return e[1] }) } logger && logger.emit('files-queued', queued, index) diff --git a/ui (gen4)/lib/util.js b/ui (gen4)/lib/util.js index db305d00..5ed575a9 100755 --- a/ui (gen4)/lib/util.js +++ b/ui (gen4)/lib/util.js @@ -196,17 +196,26 @@ Array.prototype.sortAs = function(other){ // // .mapChunks(func) // .mapChunks(chunk_size, func) +// .mapChunks([item_handler, chunk_handler]) +// .mapChunks(chunk_size, [item_handler, chunk_handler]) // -> promise(list) // // .filterChunks(func) // .filterChunks(chunk_size, func) +// .filterChunks([item_handler, chunk_handler]) +// .filterChunks(chunk_size, [item_handler, chunk_handler]) // -> promise(list) // // .reduceChunks(func, res) // .reduceChunks(chunk_size, func, res) +// .reduceChunks([item_handler, chunk_handler], res) +// .reduceChunks(chunk_size, [item_handler, chunk_handler], res) // -> promise(res) // // +// chunk_handler(chunk, result, offset) +// +// // chunk_size can be: // 20 - chunk size // '20' - chunk size @@ -223,7 +232,8 @@ var makeChunkIter = function(iter, wrapper){ return function(size, func, ...rest){ var that = this var args = [...arguments] - size = args[0] instanceof Function ? + size = (args[0] instanceof Function + || args[0] instanceof Array) ? (this.CHUNK_SIZE || 50) : args.shift() size = typeof(size) == typeof('str') ? @@ -232,31 +242,38 @@ var makeChunkIter = function(iter, wrapper){ Math.round(this.length / (parseInt(size) || 1)) || 1 : parseInt(size)) : size + var postChunk func = args.shift() + ;[func, postChunk] = func instanceof Array ? func : [func] rest = args var res = [] var _wrapper = wrapper.bind(this, res, func, this) return new Promise(function(resolve, reject){ - var next = function(chunks){ - setTimeout(function(){ - res.push( - chunks.shift()[iter](_wrapper, ...rest)) - // stop condition... - chunks.length == 0 ? - resolve(res.flat(2)) - : next(chunks) }, 0) } - next(that - // split the array into chunks... - .reduce(function(res, e, i){ - var c = res.slice(-1)[0] - c.length >= size ? - // initial element in chunk... - res.push([[i, e]]) - // rest... - : c.push([i, e]) - return res }, [[]])) - }) } } + var next = function(chunks){ + setTimeout(function(){ + var chunk, val + res.push( + val = (chunk = chunks.shift())[iter](_wrapper, ...rest)) + postChunk + && postChunk.call(that, + chunk.map(function([i, v]){ return v }), + val, + chunk[0][0]) + // stop condition... + chunks.length == 0 ? + resolve(res.flat(2)) + : next(chunks) }, 0) } + next(that + // split the array into chunks... + .reduce(function(res, e, i){ + var c = res.slice(-1)[0] + c.length >= size ? + // initial element in chunk... + res.push([[i, e]]) + // rest... + : c.push([i, e]) + return res }, [[]])) }) } } Array.prototype.CHUNK_SIZE = 50 Array.prototype.mapChunks = makeChunkIter('map')