diff --git a/ui (gen4)/features/filesystem.js b/ui (gen4)/features/filesystem.js index 28c58d07..5a64b192 100755 --- a/ui (gen4)/features/filesystem.js +++ b/ui (gen4)/features/filesystem.js @@ -765,11 +765,18 @@ var FileSystemLoaderActions = actions.Actions({ checkIndex: ['File/Check index consistency', core.doc`Check index consistency... + Check currently loaded index for missing references and fix them + if found. + This will: - remove references to non-existing preview images (image.preview) - remove references to non-existing .path (image.path) - if .path removed, set to largest available preview + + NOTE: currently this is disabled for merged indexes, need to load + and check individually... + `, function(logger){ var that = this @@ -786,11 +793,15 @@ var FileSystemLoaderActions = actions.Actions({ .forEach(function(gid){ logger.emit('queued', gid)}) + // XXX get this from config... + var chunk_size = 50 - /* // XXX this version of the code blocks the ui until it's - // done... return this.images - .map(function(gid, image){ + .map(function(gid, image){ + return [gid, image] }) + .mapChunks(chunk_size, function(e){ + var gid = e[0] + var image = e[1] var updated = false var previews = image.preview || {} @@ -808,57 +819,8 @@ var FileSystemLoaderActions = actions.Actions({ return updated ? gid : [] }) - .flat() - /*/ - var chunk_size = 50 - return new Promise(function(resolve, reject){ - var res = [] - that.images - .reduce(function(res, e, gid){ - var c = res.slice(-1)[0] - c.length < chunk_size ? - c.push([gid, e]) - : res.push([[gid, e]]) - return res - }, [[]]) - .map(function(chunk, i, chunks){ - setTimeout(function(){ - res.push(chunk - // NOTE: all this complication with promises is - // needed to let the ui a chance to show - // progress... - // NOTE: if a better way is found this is the - // only part needed, just iterate over - // return this.images - // .map(function(gid, image){ - // // place the func body here... - // ... - // }) - .map(function(v){ - var gid = v[0] - var image = v[1] - var updated = false - - var previews = image.preview || {} - Object.entries(previews) - .forEach(function(p){ - !fse.existsSync(image.base_path +'/'+ p[1]) - && (updated = true) - && (delete previews[p[0]]) }) - - !fse.existsSync(image.base_path +'/'+ image.path) - && (updated = true) - && (delete image.path) - - logger && logger.emit('done', gid) - - return updated ? gid : [] - })) - - i >= chunks.length-1 - && resolve(res.flat(Infinity)) - }, 0) }) }) - //*/ + .then(function(res){ + return res.flat() }) }], }) diff --git a/ui (gen4)/lib/util.js b/ui (gen4)/lib/util.js index ea125d37..6d81283e 100755 --- a/ui (gen4)/lib/util.js +++ b/ui (gen4)/lib/util.js @@ -198,6 +198,84 @@ Array.prototype.sortAs = function(other){ +// Equivalent to .map(..) / .filter(..) / .reduce(..) / .forEach(..) that +// process the contents in chunks asynchronously... +// +// .mapChunks(func) +// .mapChunks(chunk_size, func) +// -> promise(list) +// +// .filterChunks(func) +// .filterChunks(chunk_size, func) +// -> promise(list) +// +// .reduceChunks(func, res) +// .reduceChunks(chunk_size, func, res) +// -> promise(res) +// +// +var makeChunkIter = function(iter, wrapper){ + wrapper = wrapper + || function(res, func, array, e){ + return func.call(this, e[1], e[0], array) } + return function(size, func, ...rest){ + var that = this + var args = [...arguments] + size = args[0] instanceof Function ? + (this.chunk_size || 50) + : args.shift() + func = args.shift() + rest = args + var res = [] + var _wrapper = wrapper.bind(this, res, func, this) + + return new Promise(function(resolve, reject){ + that + // split the array into chunks... + .reduce(function(res, e, i){ + var c = res.slice(-1)[0] + c.length >= size ? + // initial element in chunk... + res.push([[i, e]]) + // rest... + : c.push([i, e]) + return res + }, [[]]) + // go through each chunk async... + .forEach(function(chunk, i, chunks){ + setTimeout(function(){ + res.push( + // NOTE: all this complication with promises is + // needed to let the ui a chance to show + // progress... + // NOTE: if a better way is found this is the + // only part needed, just iterate over + // return this.images + // .map(function(gid, image){ + // // place the func body here... + // ... + // }) + chunk[iter](_wrapper, ...rest)) + + i >= chunks.length-1 + && resolve(res.flat(2)) + }, 0) }) }) } } + +Array.prototype.chunk_size = 50 +Array.prototype.mapChunks = makeChunkIter('map') +Array.prototype.filterChunks = makeChunkIter('map', + function(res, func, array, e){ + return !!func.call(this, e[1], e[0], array) ? [e[1]] : [] }) +Array.prototype.reduceChunks = makeChunkIter('reduce', + function(total, func, array, res, e){ + return func.call(this, + total.length > 0 ? + total.pop() + : res, + e[1], e[0], array) }) + + + //--------------------------------------------------------------------- // Set...