Package Products :: Package ZenUtils :: Module MultiPathIndex
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenUtils.MultiPathIndex

  1  ########################################################################### 
  2  # 
  3  # This program is part of Zenoss Core, an open source monitoring platform. 
  4  # Copyright (C) 2008, Zenoss Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify it 
  7  # under the terms of the GNU General Public License version 2 as published by 
  8  # the Free Software Foundation. 
  9  # 
 10  # For complete information please visit: http://www.zenoss.com/oss/ 
 11  # 
 12  ########################################################################### 
 13  import time 
 14  import logging 
 15  LOG = logging.getLogger('ZenUtils.MultiPathIndex') 
 16   
 17  from Globals import DTMLFile 
 18   
 19  from ExtendedPathIndex import ExtendedPathIndex 
 20  from Products.PluginIndexes.common import safe_callable 
 21  from BTrees.OOBTree import OOSet 
 22  from BTrees.IIBTree import IISet, intersection, union, multiunion 
 23   
24 -def _isSequenceOfSequences(seq):
25 if not seq: 26 return False 27 return (isinstance(seq, (tuple, list)) and 28 all(isinstance(item, (tuple, list)) for item in seq))
29
30 -def _recursivePathSplit(seq):
31 if isinstance(seq, (tuple, list)): 32 return map(_recursivePathSplit, seq) 33 if '/' in seq: 34 return seq.split('/') 35 else: 36 return seq
37 38
39 -class MultiPathIndex(ExtendedPathIndex):
40 """ 41 A path index that is capable of indexing multiple paths per object. 42 """ 43 meta_type = "MultiPathIndex" 44 45
46 - def search(self, path, default_level=0, depth=-1, navtree=0, 47 navtree_start=0):
48 """ 49 path is either a string representing a 50 relative URL or a part of a relative URL or 51 a tuple (path,level). 52 53 level >= 0 starts searching at the given level 54 level < 0 not implemented yet 55 """ 56 57 if isinstance(path, basestring): 58 startlevel = default_level 59 else: 60 startlevel = int(path[1]) 61 path = path[0] 62 63 absolute_path = isinstance(path, basestring) and path.startswith('/') 64 comps = filter(None, path.split('/')) 65 66 orig_comps = [''] + comps[:] 67 68 if depth > 0: 69 raise ValueError, "Can't do depth searches anymore" 70 71 if not comps: 72 comps = ['dmd'] 73 startlevel = 1 74 elif comps[0] == 'zport': 75 comps = comps[1:] 76 elif comps[0] != 'dmd': 77 raise ValueError, "Depth searches must start with 'dmd'" 78 startlevel = len(comps) 79 #startlevel = len(comps)-1 if len(comps) > 1 else 1 80 81 if len(comps) == 0: 82 if depth == -1 and not navtree: 83 return IISet(self._unindex.keys()) 84 85 # Make sure that we get depth = 1 if in navtree mode 86 # unless specified otherwise 87 88 orig_depth = depth 89 if depth == -1: 90 depth = 0 or navtree 91 92 # Optimized navtree starting with absolute path 93 if absolute_path and navtree and depth == 1 and default_level==0: 94 set_list = [] 95 # Insert root element 96 if navtree_start >= len(orig_comps): 97 navtree_start = 0 98 # create a set of parent paths to search 99 for i in range(len(orig_comps), navtree_start, -1): 100 parent_path = '/'.join(orig_comps[:i]) 101 parent_path = parent_path and parent_path or '/' 102 try: 103 set_list.append(self._index_parents[parent_path]) 104 except KeyError: 105 pass 106 return multiunion(set_list) 107 # Optimized breadcrumbs 108 elif absolute_path and navtree and depth == 0 and default_level==0: 109 item_list = IISet() 110 # Insert root element 111 if navtree_start >= len(orig_comps): 112 navtree_start = 0 113 # create a set of parent paths to search 114 for i in range(len(orig_comps), navtree_start, -1): 115 parent_path = '/'.join(orig_comps[:i]) 116 parent_path = parent_path and parent_path or '/' 117 try: 118 item_list.insert(self._index_items[parent_path]) 119 except KeyError: 120 pass 121 return item_list 122 # Specific object search 123 elif absolute_path and orig_depth == 0 and default_level == 0: 124 try: 125 return IISet([self._index_items[path]]) 126 except KeyError: 127 return IISet() 128 # Single depth search 129 elif absolute_path and orig_depth == 1 and default_level == 0: 130 # only get objects contained in requested folder 131 try: 132 return self._index_parents[path] 133 except KeyError: 134 return IISet() 135 # Sitemaps, relative paths, and depth queries 136 elif startlevel >= 0: 137 138 pathset = None # Same as pathindex 139 navset = None # For collecting siblings along the way 140 depthset = None # For limiting depth 141 142 if navtree and depth and \ 143 self._index.has_key(None) and \ 144 self._index[None].has_key(startlevel): 145 navset = self._index[None][startlevel] 146 for level in range(startlevel, startlevel+len(comps)): 147 if level <= len(comps): 148 comp = "/".join(comps[:level]) 149 if (not self._index.has_key(comp) 150 or not self._index[comp].has_key(level)): 151 # Navtree is inverse, keep going even for 152 # nonexisting paths 153 if navtree: 154 pathset = IISet() 155 else: 156 return IISet() 157 else: 158 return self._index[comp][level] 159 if navtree and depth and \ 160 self._index.has_key(None) and \ 161 self._index[None].has_key(level+depth): 162 navset = union(navset, intersection(pathset, 163 self._index[None][level+depth])) 164 if level-startlevel >= len(comps) or navtree: 165 if (self._index.has_key(None) 166 and self._index[None].has_key(level)): 167 depthset = union(depthset, intersection(pathset, 168 self._index[None][level])) 169 170 if navtree: 171 return union(depthset, navset) or IISet() 172 elif depth: 173 return depthset or IISet() 174 else: 175 return pathset or IISet() 176 177 else: 178 results = IISet() 179 for level in range(0,self._depth + 1): 180 ids = None 181 error = 0 182 for cn in range(0,len(comps)): 183 comp = comps[cn] 184 try: 185 ids = intersection(ids,self._index[comp][level+cn]) 186 except KeyError: 187 error = 1 188 if error==0: 189 results = union(results,ids) 190 return results
191 192
193 - def getIndexSourceNames(self):
194 """ return names of indexed attributes """ 195 return (self.id, )
196
197 - def index_object(self, docid, obj, threshold=100):
198 """ hook for (Z)Catalog """ 199 200 f = getattr(obj, self.id, None) 201 if f is not None: 202 if safe_callable(f): 203 try: 204 paths = f() 205 except AttributeError: 206 return 0 207 else: 208 paths = f 209 else: 210 try: 211 paths = obj.getPhysicalPath() 212 except AttributeError: 213 return 0 214 215 if not paths: return 0 216 paths = _recursivePathSplit(paths) 217 if not _isSequenceOfSequences(paths): 218 paths = [paths] 219 220 if docid in self._unindex: 221 unin = self._unindex[docid] 222 # Migrate old versions of the index to use OOSet 223 if isinstance(unin, set): 224 unin = self._unindex[docid] = OOSet(unin) 225 for oldpath in list(unin): 226 if list(oldpath.split('/')) not in paths: 227 self.unindex_paths(docid, (oldpath,)) 228 else: 229 self._unindex[docid] = OOSet() 230 self._length.change(1) 231 232 self.index_paths(docid, paths) 233 234 return 1
235 236
237 - def index_paths(self, docid, paths):
238 for path in paths: 239 if isinstance(path, (list, tuple)): 240 path = '/'+ '/'.join(path[1:]) 241 comps = filter(None, path.split('/')) 242 parent_path = '/' + '/'.join(comps[:-1]) 243 244 for i in range(len(comps)): 245 comp = "/".join(comps[1:i+1]) 246 if comp: 247 self.insertEntry(comp, docid, i) 248 249 # Add terminator 250 self.insertEntry(None, docid, len(comps)-1, parent_path, path) 251 252 self._unindex.setdefault(docid, OOSet()).insert(path)
253 254
255 - def unindex_paths(self, docid, paths):
256 257 if not self._unindex.has_key(docid): 258 return 259 260 def unindex(comp, level, docid=docid, parent_path=None, 261 object_path=None): 262 try: 263 self._index[comp][level].remove(docid) 264 265 if not self._index[comp][level]: 266 del self._index[comp][level] 267 268 if not self._index[comp]: 269 del self._index[comp] 270 # Remove parent_path and object path elements 271 if parent_path is not None: 272 self._index_parents[parent_path].remove(docid) 273 if not self._index_parents[parent_path]: 274 del self._index_parents[parent_path] 275 if object_path is not None: 276 del self._index_items[object_path] 277 except KeyError: 278 # Failure 279 pass
280 281 old = set(self._unindex.get(docid, ())) 282 mkstr = lambda path:'/'.join(path) if isinstance(path, tuple) else path 283 paths = map(mkstr, paths) 284 toremove = set(paths) & old 285 tokeep = old - toremove 286 for path in toremove: 287 if not path.startswith('/'): 288 path = '/'+path 289 comps = path.split('/') 290 parent_path = '/'.join(comps[:-1]) 291 292 for level in range(1, len(comps[2:])+1): 293 comp = "/".join(comps[2:level+2]) 294 unindex(comp, level, docid, parent_path, path) 295 # Remove the terminator 296 level = len(comps[1:]) 297 comp = None 298 unindex(comp, level-1, parent_path=parent_path, object_path=path) 299 300 self._unindex[docid].remove(path) 301 302 if tokeep: 303 self.index_paths(docid, tokeep) 304 else: 305 # Cleared out all paths for the object 306 self._length.change(-1) 307 del self._unindex[docid]
308 309
310 - def unindex_object(self, docid):
311 """ hook for (Z)Catalog """ 312 if not self._unindex.has_key(docid): 313 return 314 self.unindex_paths(docid, self._unindex[docid])
315 316 manage = manage_main = DTMLFile('dtml/manageMultiPathIndex', globals()) 317 manage_main._setName('manage_main') 318 319 320 manage_addMultiPathIndexForm = DTMLFile('dtml/addMultiPathIndex', globals()) 321
322 -def manage_addMultiPathIndex(self, id, REQUEST=None, RESPONSE=None, 323 URL3=None):
324 """ 325 Add a MultiPathIndex. 326 """ 327 return self.manage_addIndex(id, 'MultiPathIndex', extra=None, 328 REQUEST=REQUEST, RESPONSE=RESPONSE, 329 URL1=URL3)
330