new tags + migrated most of the code.... collection loading still broken...

Signed-off-by: Alex A. Naanou <alex.nanou@gmail.com>

ui (gen4)/features/base.js

@@ -1177,11 +1177,7 @@ module.makeTagWalker =
 function(direction, dfl_tag){
 	var meth = direction == 'next' ? 'nextImage' : 'prevImage'
 	return function(tag, mode){
-		this[meth](
+		this[meth](this.data.tags.values(tag || dfl_tag), mode) } }
-			this.data.version >= '3.1' ?
-				this.data.tags.values(tag || dfl_tag)
-				: (this.data.tags || {})[tag || dfl_tag] || [],
-			mode) } }
 
 
 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
@@ -1381,33 +1377,36 @@ module.TagsEdit = core.ImageGridFeatures.Feature({
 
 				// XXX should we save an empty list *iff* changes.marked is true???
 				if(changes === true || changes.marked){
-					res.index.marked = (res.raw.data.tags || {}).marked || []
+					res.index.marked = 
+						(res.raw.data.tags.tags || {}).marked || []
 				}
 				// XXX should we save an empty list *iff* changes.bookmarked is true???
 				if(changes === true || changes.bookmarked){
 					res.index.bookmarked = [
-						(res.raw.data.tags || {}).bookmark || [],
+						(res.raw.data.tags.tags || {}).bookmark || [],
 						{},
 					]
 				}
 
 				// cleanup...
 				if(res.index.data && res.index.data.tags){
-					delete res.index.data.tags.marked
+					delete res.index.data.tags.tags.marked
-					delete res.index.data.tags.bookmark
+					delete res.index.data.tags.tags.bookmark
-					//delete res.index.data.tags.bookmark_data
 					delete res.index.data.tags
 				}
 			}],
+		// merge the tags into data...
+		['prepareIndexForLoad.pre',
+			function(json){
+				// NOTE: this is done before we build the data to let 
+				// 		Data handle format conversion...
+				json.data.tags = json.tags || {}
+			}],
+		// merge in marked and bookmark tags...
 		['prepareIndexForLoad',
 			function(res, json){
-				res.data.tags = json.tags || {}
+				res.data.tag('marked', json.marked || [])
-
+				res.data.tag('bookmark', json.bookmarked ? json.bookmarked[0] : [])
-				res.data.tags.marked = json.marked || []
-				res.data.tags.bookmark = json.bookmarked ? json.bookmarked[0] : []
-				//res.data.tags.bookmark_data = json.bookmarked ? json.bookmarked[1] : {}
-
-				res.data.sortTags()
 			}],
 	],
 })
@@ -1794,22 +1793,9 @@ module.CropActions = actions.Actions({
 		}],
 	
 	// XXX should this be here???
-	/*
 	cropTagged: ['- Tag|Crop/Crop tagged images',
 		function(query, flatten){
 			return this.crop(this.data.tagQuery(query), flatten) }],
-	//*/
-	//*
-	cropTagged: ['- Tag|Crop/Crop tagged images',
-		function(tags, mode, flatten){
-			if(this.data.length == 0){
-				return
-			}
-			var selector = mode == 'any' ? 'getTaggedByAny' : 'getTaggedByAll'
-			this.crop(this.data[selector](tags), flatten)
-		}],
-	//*/
-
 
 	// crop edit actions...
 	// XXX BUG? order does odd things...

ui (gen4)/features/collections.js

@@ -1916,11 +1916,11 @@ var CollectionTagsActions = actions.Actions({
 	},
 
 	collectTagged: ['- Collections|Tag/',
-		function(tags, collection){
+		function(query, collection){
-			return this.collect(this.data.getTaggedByAll(tags), collection) }],
+			return this.collect(this.data.tagQuery(query), collection) }],
 	uncollectTagged: ['- Collections|Tag/',
-		function(tags, collection){
+		function(query, collection){
-			return this.uncollect(this.data.getTaggedByAll(tags), collection) }],
+			return this.uncollect(this.data.tagQuery(query), collection) }],
 })
 
 var CollectionTags = 
@@ -1971,7 +1971,7 @@ module.CollectionTags = core.ImageGridFeatures.Feature({
 			function(title){
 				var that = this
 				var local_tag_names = this.config['collection-local-tags'] || []
-				var tags = this.data.tags
+				var tags = this.data.tags.__index
 
 				// NOTE: this is done at the .pre stage as we need to grab 
 				// 		the tags BEFORE the data gets cleared (in the case 
@@ -1982,14 +1982,12 @@ module.CollectionTags = core.ImageGridFeatures.Feature({
 					// load local_tags...
 					local_tag_names
 						.forEach(function(tag){ 
-							tags[tag] = local_tags[tag] || [] 
+							tags[tag] = new Set(local_tags[tag] || [])
 						})
 
 					;(this.crop_stack || [])
-						.forEach(function(d){ d.tags = tags })
+						.forEach(function(d){ d.tags.__index = tags })
-					this.data.tags = tags
+					this.data.tags.__index = tags
-
-					this.data.sortTags()
 				}
 			}],
 		// remove tags from unloaded collections...
@@ -2021,11 +2019,11 @@ module.CollectionTags = core.ImageGridFeatures.Feature({
 					local_tag_names
 						.forEach(function(tag){ 
 							local_tags[tag] = (!new_set || title == MAIN_COLLECTION_TITLE) ? 
-								((that.data.tags || {})[tag] || []) 
+								((that.data.tags.__index || {})[tag] || new Set()) 
-								: []
+								: new Set()
 						})
 
-					delete (this.collections[title].data || {}).tags
+					delete (this.collections[title].data || {}).__tags || {}
 				}
 			}],
 		// prevent .uncollect(..) from removing global tags...
@@ -2045,7 +2043,6 @@ module.CollectionTags = core.ImageGridFeatures.Feature({
 						tags[tag] = that.data.makeSparseImages(tags[tag], true) })
 
 					this.data.tags = tags
-					this.data.sortTags()
 				}
 			}],
 		// save .local_tags to json...
@@ -2065,35 +2062,33 @@ module.CollectionTags = core.ImageGridFeatures.Feature({
 				if(mode == 'base' 
 						&& this.collection != null
 						&& this.collection != MAIN_COLLECTION_TITLE){
-					var tags = this.data.tags || {}
+					var tags = this.data.tags.json()
 					var ltags = c[MAIN_COLLECTION_TITLE].local_tags || {}
 					var rtags = res.data.tags = {}
 
 					// move all the tags...
-					Object.keys(tags)
+					Object.keys(tags.tags)
 						.filter(function(tag){ return ltags[tag] == null })
-						.forEach(function(tag){ rtags[tag] = tags[tag].compact() })
+						.forEach(function(tag){ rtags[tag] = tags.tags[tag] })
 					// overwrite the local tags for the base...
 					Object.keys(ltags)
-						.forEach(function(tag){ rtags[tag] = ltags[tag].compact() })
+						.forEach(function(tag){ rtags[tag] = ltags[tag] })
 				}
 
-				// clear and compact tags for all collections...
+				// clear tags for all collections...
 				rc
 					&& Object.keys(rc || {})
 						// XXX skip unloaded collections...
 						.filter(function(title){ return !!rc[title].data })
 						.forEach(function(title){
-							var tags = c[title].local_tags || {}
+							// convert sets to arrays...
-							var rtags = {}
+							var local_tags = {}
-
+							Object.entries(c[title].local_tags || {})
-							// compact the local tags...
+								.forEach(function(e){
-							Object.keys(tags)
+									local_tags[e[0]] = [...e[1]]
-								.forEach(function(tag){
+								})
-									rtags[tag] = tags[tag].compact() })
-
 							// move .local_tags to .data.tags
-							rc[title].data.tags = rtags
+							rc[title].data.tags.tags = local_tags
 						})
 			}],
 		// load collection local tags from .data.tags to .local_tags...
@@ -2117,9 +2112,10 @@ module.CollectionTags = core.ImageGridFeatures.Feature({
 							c.local_tags = c.local_tags || {}
 							;(that.config['collection-local-tags'] || [])
 								.forEach(function(tag){
-									c.local_tags[tag] = c.local_tags[tag] || t[tag] || [] })
+									c.local_tags[tag] = new Set(c.local_tags[tag] || t[tag] || []) })
 						})
 			}],
+		//*/
 	],
 })
 
@@ -2216,9 +2212,7 @@ var AutoCollectionsActions = actions.Actions({
 					.filter(function(tag){ 
 						return local_tag_names.indexOf(tag) < 0 })
 
-				// XXX should this be a real tag query???
+				var gids = this.data.tagQuery(tags)
-				//var gids = this.data.getTaggedByAll(tags, true)
-				var gids = this.data.getTaggedByAll(tags)
 
 				// get items that topped matching the query...
 				var remove = state.data ?

ui (gen4)/features/filesystem.js

@@ -279,6 +279,8 @@ var FileSystemLoaderActions = actions.Actions({
 			//file.loadIndex(path, this.config['index-dir'], logger)
 			return file.loadIndex(path, index_dir, from_date, logger)
 				.then(function(res){
+					var force_full_save = false
+
 					// XXX if res is empty load raw...
 
 					// XXX use the logger...
@@ -333,6 +335,14 @@ var FileSystemLoaderActions = actions.Actions({
 							}
 						}
 
+						// prepare to do a full save if format version updated...
+						if(res[k].data.version != that.data.version){
+							logger && logger.emit('Date version changed:',
+								res[k].data.version, '->', that.data.version)
+
+							force_full_save = true
+						}
+
 						var part = that.prepareIndexForLoad(res[k], k)
 
 						// load the first index...
@@ -384,6 +394,10 @@ var FileSystemLoaderActions = actions.Actions({
 					// this is the critical section, after this point we
 					// are doing the actual loading....
 					that.loadOrRecover(index) 
+						.then(function(){
+							force_full_save
+								&& that.markChanged('all')
+						})
 				})
 		}],
 
@@ -580,7 +594,7 @@ var FileSystemLoaderActions = actions.Actions({
 							}
 						})
 						.then(function(){
-							delete that.changes
+							that.markChanged('none')
 						})
 				})
 		}],

ui (gen4)/features/marks.js

@@ -229,15 +229,9 @@ var ImageMarkActions = actions.Actions({
 	// 		the problem is that on large sets this may take up quite a 
 	// 		chunk of memory...
 	get marked(){
-		if(this.data == null 
+		return this.data == null ?
-				|| this.data.tags == null){
+			[]
-			return []
+			: this.data.tagQuery('marked') },
-		}
-		// XXX remove the version test here....
-		return this.data.version >= '3.1' ?
-			this.data.tagQuery('marked')
-			: this.data.getImages((this.data.tags || {marked:[]})['marked'])
-	},
 
 	markedInRibbon: ['- Mark|Ribbon/',
 		function(ribbon){
@@ -261,7 +255,8 @@ var ImageMarkActions = actions.Actions({
 	cropMarked: ['Mark|Crop/Crop $marked images',
 		{browseMode: function(target){ 
 			return this.marked.length == 0 && 'disabled' }},
-		function(flatten){ this.cropTagged('marked', 'any', flatten) }],
+		function(flatten){ this.cropTagged('marked', flatten) }],
+		//function(flatten){ this.cropTagged('marked', 'any', flatten) }],
 
 	removeMarkedFromCrop: ['Mark|Crop/Remove marked from crop',
 		{browseMode: function(target){ 
@@ -376,14 +371,11 @@ var ImageMarkEditActions = actions.Actions({
 		'toggleMark: "loaded" "on"' ],
 
 	markTagged: ['- Mark/Mark images by tags',
-		function(tags, mode){
+		function(query){
-			var selector = mode == 'any' ? 'getTaggedByAny' : 'getTaggedByAll'
-
 			var that = this
-			this.data[selector](tags).forEach(function(gid){
+			this.data.tagQuery(query)
-				that.toggleMark(gid, 'on')
+				.forEach(function(gid){
-			})
+					that.toggleMark(gid, 'on') }) }],
-		}],
 
 	shiftMarkedUp: ['Mark/Shift marked u$p',
 		{undo: undoShift('shiftMarkedDown'),
@@ -509,16 +501,9 @@ var ImageBookmarkActions = actions.Actions({
 	// 		the problem is that on large sets this may take up quite a 
 	// 		chunk of memory...
 	get bookmarked(){
-		if(this.data == null 
+		return this.data == null ?
-				|| this.data.tags == null){
+			[]
-			return []
+			: this.data.tagQuery('bookmark') },
-		}
-		//return this.data.tags['bookmark'].slice()
-		// XXX remove the version test here....
-		return this.data.version >= '3.1' ?
-			this.data.tagQuery('bookmark')
-			: this.data.getImages((this.data.tags || {bookmark:[]})['bookmark'])
-	},
 
 	prevBookmarked: ['Bookmark|Navigate/Previous bookmarked image',
 		{browseMode: function(target){ 
@@ -532,7 +517,8 @@ var ImageBookmarkActions = actions.Actions({
 	cropBookmarked: ['Bookmark|Crop/Crop $bookmarked images',
 		{browseMode: function(target){ 
 			return this.bookmarked.length == 0 && 'disabled' }},
-		function(flatten){ this.cropTagged('bookmark', 'any', flatten) }],
+		//function(flatten){ this.cropTagged('bookmark', 'any', flatten) }],
+		function(flatten){ this.cropTagged('bookmark', flatten) }],
 })
 
 // NOTE: this is usable without ribbons...
@@ -569,8 +555,7 @@ var ImageBookmarkEditActions = actions.Actions({
 	toggleBookmarkOnMarked: ['Bookmark|Mark/-70:Toggle bookmark on maked images',
 		{browseMode: 'cropMarked'},
 		function(action){ 
-			return this.toggleBookmark(this.data.getTaggedByAny('marked'), action) 
+			return this.toggleBookmark(this.marked, action) }],
-		}],
 })
 
 var ImageBookmarksEdit = 

ui (gen4)/features/meta.js

@@ -177,6 +177,8 @@ core.ImageGridFeatures.Feature('imagegrid-testing', [
 	//----------------------------------------------------- testing ---
 	'experiments',
 	'-tests',
+	// XXX this is really slow on load, need to speed the search up...
+	'-comments',
 
 	// missing suggested feature test -- should show up in .features.missing...
 	'missing-feature',

ui (gen4)/features/ui-status.js

@@ -751,7 +751,7 @@ module.StatusBar = core.ImageGridFeatures.Feature({
 			function(){
 				this.toggleStatusBar(this.config['status-bar'])
 			}],
-		['focusImage clear markChanged',
+		['focusImage clear markChanged refresh',
 			function(){
 				this.updateStatusBar()
 			}],

ui (gen4)/imagegrid/data.js

@@ -28,9 +28,7 @@ var formats = require('imagegrid/formats')
 // 		...this is done to gradually migrate to new format versions with
 // 		minimal changes.
 var DATA_VERSION =
-// XXX 3.1 not ready for production yet...
+module.DATA_VERSION = '3.1'
-//module.DATA_VERSION = '3.1'
-module.DATA_VERSION = '3.0'
 
 
 
@@ -3304,6 +3302,10 @@ var DataWithTagsPrototype = {
 			res 
 			: this.getImages(res)
 	},
+
+	// XXX re-implement the above in this...
+	tagQuery: function(query){
+		throw Error('.tagQuery(..): Not implemented.') },
 }
 
 
@@ -3317,6 +3319,11 @@ var DataWithTags2Prototype = {
 	set tags(value){
 		this.__tags = value },
 
+	get untagged(){
+		var v = new Set(this.tags.values())
+		return this.getImages()
+			.filter(function(gid){ return !v.has(gid) }) },
+
 	// XXX do we need these???
 	hasTag: function(gid, ...tags){
 		return this.tags.tags(this.getImage(gid), ...tags) },
@@ -3382,21 +3389,6 @@ var DataWithTags2Prototype = {
 	tagsToImages: function(){
 		throw Error('.tagsToImages(..): Not implemented.') },
 
-
-	// XXX compatibility...
-	// XXX check if these are ever used without raw...
-	getTaggedByAll: function(tags, raw){
-		var res = this.tags.query(['and', ...tags])
-		return raw ? 
-			res 
-			: this.getImages(res) },
-	getTaggedByAny: function(tags, raw){
-		var res = this.tags.query(['or', ...tags])
-		return raw ? 
-			res 
-			: this.getImages(res) },
-
-
 	// Extended methods...
 	//
 	// special case: make the tags mutable...
@@ -3405,18 +3397,18 @@ var DataWithTags2Prototype = {
 		crop.tags = this.tags
 		return crop
 	},
-	// XXX
+	join: function(...others){
-	join: function(){
 		var res = DataWithTags2Prototype.__proto__.join.apply(this, arguments)
-		// XXX
+		res.tags.join(...others
-		throw Error('.join(..): Not implemented.')
+			.map(function(other){ 
+				return other.tags }))
 		return res
 	},
-	// XXX
+	// XXX should this account for crop???
+	// XXX test...
 	split: function(){
 		var res = DataWithTags2Prototype.__proto__.split.apply(this, arguments)
-		// XXX
+		res.tags = res.tags.filter(res.order)
-		throw Error('.split(..): Not implemented.')
 		return res
 	},
 	clone: function(){
@@ -3474,10 +3466,7 @@ var DataWithTags =
 module.DataWithTags = 
 object.makeConstructor('DataWithTags', 
 		DataClassPrototype, 
-		// XXX remove the version test here....
+		DataWithTags2Prototype)
-		DATA_VERSION >= '3.1' ?
-			DataWithTags2Prototype
-			: DataWithTagsPrototype)
 
 
 var Data =

ui (gen4)/imagegrid/tags.js

@@ -752,6 +752,27 @@ var TagsPrototype = {
 	},
 
 
+	join: function(...others){
+		var that = this
+		var index = this.__index
+		others
+			.forEach(function(other){
+				Object.entries(other.__index)
+					.forEach(function(e){
+						index[e[0]] = new Set([...(index[e[0]] || []), ...e[1]]) }) })
+		return this
+	},
+	// XXX create a new tagset with only the given values...
+	// XXX this should support a function...
+	filter: function(values){
+		var res = this.clone()
+		Object.values(res.__index)
+			.forEach(function(s){
+				values.forEach(function(v){ s.delete(v) }) })
+		return res
+	},
+
+
 	// Object utility API...
 	//
 	// 	.clone()
@@ -806,7 +827,7 @@ var TagsPrototype = {
 
 		// tags...
 		res.tags = {}
-		Object.entries(this.__index)
+		Object.entries(this.__index || {})
 			.forEach(function(e){
 				// XXX should we serialize the items here???
 				res.tags[e[0]] = [...e[1]] })