Package Products :: Package ZenUtils :: Module FileCache
[hide private]
[frames] | no frames]

Source Code for Module Products.ZenUtils.FileCache

  1  ########################################################################### 
  2  # 
  3  # This program is part of Zenoss Core, an open source monitoring platform. 
  4  # Copyright (C) 2010, Zenoss Inc. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify it 
  7  # under the terms of the GNU General Public License version 2 or (at your 
  8  # option) any later version as published by the Free Software Foundation. 
  9  # 
 10  # For complete information please visit: http://www.zenoss.com/oss/ 
 11  # 
 12  ########################################################################### 
 13  ## Based on Python recipe http://code.activestate.com/recipes/543263/ (r1) 
 14  # by Brian O. Bush, Thu 24-Jan-2008 06:53 bushbo 
 15   
 16  import os, sys, pickle, base64, threading, glob 
 17  import tempfile 
 18   
 19  _DEFAULT_NOT_SPECIFIED = object() 
 20   
 21  # This file cache is thread-safe 
22 -class FileCache(object):
23 - def __init__(self, path, protocol=-1):
24 self.path = path # path assumed existing; check externally 25 if not os.path.exists(self.path): 26 os.makedirs(self.path) 27 self.gen_key = lambda x: '%s.pickle' % base64.b64encode(x) 28 self.lock = threading.Lock() 29 self._pickleProtocol = protocol
30 - def _makeFileNameFromKey(self, key):
31 return os.path.join(self.path, self.gen_key(key))
32 - def _allFileNames(self):
33 return glob.glob(os.path.join(self.path,'*.pickle'))
34 - def get(self, key, default=_DEFAULT_NOT_SPECIFIED):
35 retval = default 36 fn = self._makeFileNameFromKey(key) 37 with self.lock: 38 try: 39 with open(fn, 'rb') as f: 40 retval = pickle.load(f) 41 except IOError: 42 if default is _DEFAULT_NOT_SPECIFIED: 43 raise KeyError('no such key ' + key) 44 else: 45 return default 46 return retval[1]
47 - def __getitem__(self, key):
48 return self.get(key)
49 - def __setitem__(self, key, value):
50 fn = self._makeFileNameFromKey(key) 51 with self.lock: 52 # use temp file to avoid partially written out pickles 53 tempFd = None # file descriptor 54 tempFn = None # file name 55 try: 56 # dump to a temp file in pickle dir 57 tempFd, tempFn = tempfile.mkstemp(dir=os.path.dirname(fn)) 58 # open a file object 59 with os.fdopen(tempFd, "wb") as tempF: 60 tempFd = None 61 pickle.dump((key, value), tempF, protocol=self._pickleProtocol) 62 # rename the temp file 63 # this is an atomic operation on most filesystems 64 os.rename(tempFn, fn) 65 except Exception as ex: 66 if tempFd is not None: 67 os.close(tempFd) 68 raise ex 69 finally: 70 if os.path.exists(tempFn): 71 try: 72 os.remove(tempFn) 73 except (OSError, IOError): 74 pass
75 - def __delitem__(self, key):
76 fn = self._makeFileNameFromKey(key) 77 with self.lock: 78 try: 79 os.remove(fn) 80 except (OSError, IOError): 81 raise KeyError('no such key ' + key)
82 - def clear(self):
83 with self.lock: 84 for fn in self._allFileNames(): 85 try: 86 os.remove(fn) 87 except (OSError, IOError): 88 pass
89 - def items(self):
90 with self.lock: 91 return list(self.iteritems())
92 - def keys(self):
93 with self.lock: 94 return list(self.iterkeys())
95 - def values(self):
96 with self.lock: 97 return list(self.itervalues())
98 - def iterkeys(self):
99 for fn in self._allFileNames(): 100 yield base64.b64decode(os.path.split(fn)[1][:-7])
101 - def itervalues(self):
102 for k,v in self.iteritems(): 103 yield v
104 - def iteritems(self):
105 for fn in self._allFileNames(): 106 try: 107 with open(fn,'rb') as f: 108 yield pickle.load(f) 109 except IOError: 110 pass
111 - def __contains__(self, key):
112 with self.lock: 113 fn = self._makeFileNameFromKey(key) 114 return os.path.exists(fn)
115 - def __len__(self):
116 with self.lock: 117 return len(self._allFileNames())
118 - def __bool__(self):
119 with self.lock: 120 return bool(self._allFileNames())
121 __nonzero__ = __bool__ 122 123 if __name__=='__main__':
124 - class Site:
125 - def __init__(self, name, hits=0):
126 self.name = name 127 self.hits = hits
128 - def __str__(self):
129 return '%s, %d hits' % (self.name, self.hits)
130 cache = FileCache('test') 131 sites = [Site('cnn.com'), Site('kd7yhr.org', 1), Site('asdf.com', 3)] 132 # We will use the site url as the key for our cache 133 # Comment out the next two lines to test cache reading 134 for site in sites: 135 cache[site.name] = site 136 testitemname = sites[-1].name 137 138 entry = cache.get(testitemname) 139 if entry: 140 print type(entry), entry 141 142 print cache.keys() 143 import glob 144 for fn in glob.glob('test/*'): 145 print fn 146 147 print testitemname in cache 148 149 del cache[testitemname] 150 print cache.keys() 151 print testitemname in cache 152 153 cache.clear() 154 print cache.keys() 155