Package Products :: Package ZenUtils :: Module ExtendedPathIndex
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenUtils.ExtendedPathIndex

  1  # Copyright (c) 2004 Zope Corporation and Plone Solutions 
  2   
  3  # ZPL 2.1 license 
  4   
  5  import logging 
  6   
  7  from Globals import DTMLFile 
  8  from BTrees.IIBTree import IISet, intersection, union, multiunion 
  9  from BTrees.OOBTree import OOBTree 
 10  from BTrees.OIBTree import OIBTree 
 11   
 12  from Products.PluginIndexes.common.util import parseIndexRequest 
 13  from Products.PluginIndexes.common import safe_callable 
 14  from Products.PluginIndexes.PathIndex.PathIndex import PathIndex 
 15   
 16  _marker = [] 
 17  logger = logging.getLogger('ExtendedPathIndex') 
 18   
19 -class ExtendedPathIndex(PathIndex):
20 """ A path index stores all path components of the physical 21 path of an object: 22 23 Internal datastructure (regular pathindex): 24 25 - a physical path of an object is split into its components 26 27 - every component is kept as a key of a OOBTree in self._indexes 28 29 - the value is a mapping 'level of the path component' to 30 'all docids with this path component on this level' 31 32 In addition 33 34 - there is a terminator (None) signifying the last component in the path 35 36 """ 37 38 meta_type = "ExtendedPathIndex" 39 40 manage_options= ( 41 {'label': 'Settings', 42 'action': 'manage_main', 43 'help': ('ExtendedPathIndex','ExtendedPathIndex_Settings.stx')}, 44 ) 45 46 query_options = ("query", "level", "operator", "depth", "navtree", 47 "navtree_start") 48
49 - def __init__(self, id, extra=None, caller=None):
50 """ ExtendedPathIndex supports indexed_attrs """ 51 PathIndex.__init__(self, id, caller) 52 53 def get(o, k, default): 54 if isinstance(o, dict): 55 return o.get(k, default) 56 else: 57 return getattr(o, k, default)
58 59 attrs = get(extra, 'indexed_attrs', None) 60 if attrs is None: 61 return 62 if isinstance(attrs, str): 63 attrs = attrs.split(',') 64 attrs = filter(None, [a.strip() for a in attrs]) 65 66 if attrs: 67 # We only index the first attribute so snip off the rest 68 self.indexed_attrs = tuple(attrs[:1])
69
70 - def clear(self):
71 PathIndex.clear(self) 72 self._index_parents = OOBTree() 73 self._index_items = OIBTree()
74
75 - def insertEntry(self, comp, id, level, parent_path=None, object_path=None):
76 """Insert an entry. 77 78 parent_path is the path of the parent object 79 80 path is the object path, it is assumed to be unique, i.e. there 81 is a one to one mapping between physical paths and docids. This 82 will be large, and is only used for breadcrumbs. 83 84 id is the docid 85 """ 86 87 PathIndex.insertEntry(self, comp, id, level) 88 89 if parent_path is not None: 90 if not self._index_parents.has_key(parent_path): 91 self._index_parents[parent_path] = IISet() 92 93 self._index_parents[parent_path].insert(id) 94 95 # We make the assumption that a full path corresponds one and only 96 # one object. 97 98 if object_path is not None: 99 self._index_items[object_path] = id
100
101 - def index_object(self, docid, obj ,threshold=100):
102 """ hook for (Z)Catalog """ 103 104 # PathIndex first checks for an attribute matching its id and 105 # falls back to getPhysicalPath only when failing to get one. 106 # The presence of 'indexed_attrs' overrides this behavior and 107 # causes indexing of the custom attribute. 108 109 attrs = getattr(self, 'indexed_attrs', None) 110 if attrs: 111 index = attrs[0] 112 else: 113 index = self.id 114 115 f = getattr(obj, index, None) 116 if f is not None: 117 if safe_callable(f): 118 try: 119 path = f() 120 except AttributeError: 121 return 0 122 else: 123 path = f 124 125 if not isinstance(path, (str, tuple)): 126 raise TypeError('path value must be string or tuple ' 127 'of strings: (%r, %s)' % (index, repr(path))) 128 else: 129 try: 130 path = obj.getPhysicalPath() 131 except AttributeError: 132 return 0 133 134 if isinstance(path, (list, tuple)): 135 path = '/'+ '/'.join(path[1:]) 136 comps = filter(None, path.split('/')) 137 parent_path = '/' + '/'.join(comps[:-1]) 138 139 # Make sure we reindex properly when path change 140 if self._unindex.has_key(docid) and self._unindex.get(docid) != path: 141 self.unindex_object(docid) 142 143 if not self._unindex.has_key(docid): 144 self._length.change(1) 145 146 for i in range(len(comps)): 147 self.insertEntry(comps[i], docid, i) 148 149 # Add terminator 150 self.insertEntry(None, docid, len(comps)-1, parent_path, path) 151 152 self._unindex[docid] = path 153 return 1
154
155 - def unindex_object(self, docid):
156 """ hook for (Z)Catalog """ 157 158 if not self._unindex.has_key(docid): 159 logger.log(logging.INFO, 160 'Attempt to unindex nonexistent object with id ' 161 '%s' % docid) 162 return 163 164 # There is an assumption that paths start with / 165 path = self._unindex[docid] 166 if not path.startswith('/'): 167 path = '/'+path 168 comps = path.split('/') 169 parent_path = '/'.join(comps[:-1]) 170 171 def unindex(comp, level, docid=docid, parent_path=None, 172 object_path=None): 173 try: 174 self._index[comp][level].remove(docid) 175 176 if not self._index[comp][level]: 177 del self._index[comp][level] 178 179 if not self._index[comp]: 180 del self._index[comp] 181 # Remove parent_path and object path elements 182 if parent_path is not None: 183 self._index_parents[parent_path].remove(docid) 184 if not self._index_parents[parent_path]: 185 del self._index_parents[parent_path] 186 if object_path is not None: 187 del self._index_items[object_path] 188 except KeyError: 189 logger.log(logging.INFO, 190 'Attempt to unindex object with id ' 191 '%s failed' % docid)
192 193 for level in range(len(comps[1:])): 194 comp = comps[level+1] 195 unindex(comp, level) 196 197 # Remove the terminator 198 level = len(comps[1:]) 199 comp = None 200 unindex(comp, level-1, parent_path=parent_path, object_path=path) 201 202 self._length.change(-1) 203 del self._unindex[docid] 204
205 - def search(self, path, default_level=0, depth=-1, navtree=0, 206 navtree_start=0):
207 """ 208 path is either a string representing a 209 relative URL or a part of a relative URL or 210 a tuple (path,level). 211 212 level >= 0 starts searching at the given level 213 level < 0 not implemented yet 214 """ 215 216 if isinstance(path, basestring): 217 startlevel = default_level 218 else: 219 startlevel = int(path[1]) 220 path = path[0] 221 222 absolute_path = isinstance(path, basestring) and path.startswith('/') 223 224 comps = filter(None, path.split('/')) 225 226 orig_comps = [''] + comps[:] 227 # Optimization - avoid using the root set 228 # as it is common for all objects anyway and add overhead 229 # There is an assumption about catalog/index having 230 # the same container as content 231 if default_level == 0: 232 indexpath = list(filter(None, self.getPhysicalPath())) 233 while min(len(indexpath), len(comps)): 234 if indexpath[0] == comps[0]: 235 del indexpath[0] 236 del comps[0] 237 startlevel += 1 238 else: 239 break 240 241 if len(comps) == 0: 242 if depth == -1 and not navtree: 243 return IISet(self._unindex.keys()) 244 245 # Make sure that we get depth = 1 if in navtree mode 246 # unless specified otherwise 247 248 orig_depth = depth 249 if depth == -1: 250 depth = 0 or navtree 251 252 # Optimized navtree starting with absolute path 253 if absolute_path and navtree and depth == 1 and default_level==0: 254 set_list = [] 255 # Insert root element 256 if navtree_start >= len(orig_comps): 257 navtree_start = 0 258 # create a set of parent paths to search 259 for i in range(len(orig_comps), navtree_start, -1): 260 parent_path = '/'.join(orig_comps[:i]) 261 parent_path = parent_path and parent_path or '/' 262 try: 263 set_list.append(self._index_parents[parent_path]) 264 except KeyError: 265 pass 266 return multiunion(set_list) 267 # Optimized breadcrumbs 268 elif absolute_path and navtree and depth == 0 and default_level==0: 269 item_list = IISet() 270 # Insert root element 271 if navtree_start >= len(orig_comps): 272 navtree_start = 0 273 # create a set of parent paths to search 274 for i in range(len(orig_comps), navtree_start, -1): 275 parent_path = '/'.join(orig_comps[:i]) 276 parent_path = parent_path and parent_path or '/' 277 try: 278 item_list.insert(self._index_items[parent_path]) 279 except KeyError: 280 pass 281 return item_list 282 # Specific object search 283 elif absolute_path and orig_depth == 0 and default_level == 0: 284 try: 285 return IISet([self._index_items[path]]) 286 except KeyError: 287 return IISet() 288 # Single depth search 289 elif absolute_path and orig_depth == 1 and default_level == 0: 290 # only get objects contained in requested folder 291 try: 292 return self._index_parents[path] 293 except KeyError: 294 return IISet() 295 # Sitemaps, relative paths, and depth queries 296 elif startlevel >= 0: 297 298 pathset = None # Same as pathindex 299 navset = None # For collecting siblings along the way 300 depthset = None # For limiting depth 301 302 if navtree and depth and \ 303 self._index.has_key(None) and \ 304 self._index[None].has_key(startlevel): 305 navset = self._index[None][startlevel] 306 307 for level in range(startlevel, startlevel+len(comps) + depth): 308 if level-startlevel < len(comps): 309 comp = comps[level-startlevel] 310 if not self._index.has_key(comp) or not self._index[comp].has_key(level): 311 # Navtree is inverse, keep going even for 312 # nonexisting paths 313 if navtree: 314 pathset = IISet() 315 else: 316 return IISet() 317 else: 318 pathset = intersection(pathset, 319 self._index[comp][level]) 320 if navtree and depth and \ 321 self._index.has_key(None) and \ 322 self._index[None].has_key(level+depth): 323 navset = union(navset, intersection(pathset, 324 self._index[None][level+depth])) 325 if level-startlevel >= len(comps) or navtree: 326 if self._index.has_key(None) and self._index[None].has_key(level): 327 depthset = union(depthset, intersection(pathset, 328 self._index[None][level])) 329 330 if navtree: 331 return union(depthset, navset) or IISet() 332 elif depth: 333 return depthset or IISet() 334 else: 335 return pathset or IISet() 336 337 else: 338 results = IISet() 339 for level in range(0,self._depth + 1): 340 ids = None 341 error = 0 342 for cn in range(0,len(comps)): 343 comp = comps[cn] 344 try: 345 ids = intersection(ids,self._index[comp][level+cn]) 346 except KeyError: 347 error = 1 348 if error==0: 349 results = union(results,ids) 350 return results
351
352 - def _apply_index(self, request, cid=''):
353 """ hook for (Z)Catalog 354 'request' -- mapping type (usually {"path": "..." } 355 additionaly a parameter "path_level" might be passed 356 to specify the level (see search()) 357 358 'cid' -- ??? 359 """ 360 361 record = parseIndexRequest(request,self.id,self.query_options) 362 if record.keys==None: return None 363 364 level = record.get("level", 0) 365 operator = record.get('operator', self.useOperator).lower() 366 depth = getattr(record, 'depth', -1) # Set to 0 or navtree later 367 # use getattr to get 0 value 368 navtree = record.get('navtree', 0) 369 navtree_start = record.get('navtree_start', 0) 370 371 # depending on the operator we use intersection of union 372 if operator == "or": set_func = union 373 else: set_func = intersection 374 375 res = None 376 for k in record.keys: 377 rows = self.search(k,level, depth, navtree, navtree_start) 378 res = set_func(res,rows) 379 380 if res: 381 return res, (self.id,) 382 else: 383 return IISet(), (self.id,)
384
385 - def getIndexSourceNames(self):
386 """ return names of indexed attributes """ 387 388 # By default PathIndex advertises getPhysicalPath even 389 # though the logic in index_object is different. 390 391 try: 392 return tuple(self.indexed_attrs) 393 except AttributeError: 394 return ('getPhysicalPath',)
395 396 index_html = DTMLFile('dtml/index', globals()) 397 manage_workspace = DTMLFile('dtml/manageExtendedPathIndex', globals()) 398 399 400 manage_addExtendedPathIndexForm = DTMLFile('dtml/addExtendedPathIndex', globals()) 401
402 -def manage_addExtendedPathIndex(self, id, extra=None, REQUEST=None, RESPONSE=None, URL3=None):
403 """Add an extended path index""" 404 return self.manage_addIndex(id, 'ExtendedPathIndex', extra=extra, 405 REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
406