Package flumotion :: Package common :: Module bundle
[hide private]

Source Code for Module flumotion.common.bundle

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_bundle -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007,2008 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  bundles of files used to implement caching over the network 
 24  """ 
 25   
 26  import StringIO 
 27  import errno 
 28  import os 
 29  import sys 
 30  import tempfile 
 31  import zipfile 
 32   
 33  from flumotion.common import errors, dag, python 
 34  from flumotion.common.python import makedirs 
 35   
 36  __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket'] 
 37  __version__ = "$Rev: 7990 $" 
 38   
 39   
40 -def rename(source, dest):
41 return os.rename(source, dest)
42 43
44 -def _win32Rename(source, dest):
45 # rename a source to dest. 46 # ignores the destination if it already exists 47 # removes source if destination already exists 48 try: 49 return os.rename(source, dest) 50 except WindowsError, e: 51 import winerror 52 if e.errno == winerror.ERROR_ALREADY_EXISTS: 53 os.unlink(source)
54 55 56 if sys.platform == 'win32': 57 rename = _win32Rename 58 59
60 -class BundledFile:
61 """ 62 I represent one file as managed by a bundler. 63 """ 64
65 - def __init__(self, source, destination):
66 self.source = source 67 self.destination = destination 68 self._last_md5sum = None 69 self._last_timestamp = None 70 self.zipped = False
71
72 - def md5sum(self):
73 """ 74 Calculate the md5sum of the given file. 75 76 @returns: the md5 sum a 32 character string of hex characters. 77 """ 78 data = open(self.source, "r").read() 79 return python.md5(data).hexdigest()
80
81 - def timestamp(self):
82 """ 83 @returns: the last modified timestamp for the file. 84 """ 85 return os.path.getmtime(self.source)
86
87 - def hasChanged(self):
88 """ 89 Check if the file has changed since it was last checked. 90 91 @rtype: boolean 92 """ 93 94 # if it wasn't zipped yet, it needs zipping, so we pretend it 95 # was changed 96 # FIXME: move this out here 97 if not self.zipped: 98 return True 99 100 timestamp = self.timestamp() 101 # if file still has an old timestamp, it hasn't changed 102 # FIXME: looks bogus, shouldn't this check be != instead of <= ? 103 if self._last_timestamp and timestamp <= self._last_timestamp: 104 return False 105 self._last_timestamp = timestamp 106 107 # if the md5sum has changed, it has changed 108 md5sum = self.md5sum() 109 if self._last_md5sum != md5sum: 110 self._last_md5sum = md5sum 111 return True 112 113 return False
114
115 - def pack(self, zip):
116 self._last_timestamp = self.timestamp() 117 self._last_md5sum = self.md5sum() 118 zip.write(self.source, self.destination) 119 self.zipped = True
120 121
122 -class Bundle:
123 """ 124 I am a bundle of files, represented by a zip file and md5sum. 125 """ 126
127 - def __init__(self, name):
128 self.zip = None 129 self.md5sum = None 130 self.name = name
131
132 - def setZip(self, zip):
133 """ 134 Set the bundle to the given data representation of the zip file. 135 """ 136 self.zip = zip 137 self.md5sum = python.md5(self.zip).hexdigest()
138
139 - def getZip(self):
140 """ 141 Get the bundle's zip data. 142 """ 143 return self.zip
144 145
146 -class Unbundler:
147 """ 148 I unbundle bundles by unpacking them in the given directory 149 under directories with the bundle's md5sum. 150 """ 151
152 - def __init__(self, directory):
153 self._undir = directory
154
155 - def unbundlePathByInfo(self, name, md5sum):
156 """ 157 Return the full path where a bundle with the given name and md5sum 158 would be unbundled to. 159 """ 160 return os.path.join(self._undir, name, md5sum)
161
162 - def unbundlePath(self, bundle):
163 """ 164 Return the full path where this bundle will/would be unbundled to. 165 """ 166 return self.unbundlePathByInfo(bundle.name, bundle.md5sum)
167
168 - def unbundle(self, bundle):
169 """ 170 Unbundle the given bundle. 171 172 @type bundle: L{flumotion.common.bundle.Bundle} 173 174 @rtype: string 175 @returns: the full path to the directory where it was unpacked 176 """ 177 directory = self.unbundlePath(bundle) 178 179 filelike = StringIO.StringIO(bundle.getZip()) 180 zipFile = zipfile.ZipFile(filelike, "r") 181 zipFile.testzip() 182 183 filepaths = zipFile.namelist() 184 for filepath in filepaths: 185 path = os.path.join(directory, filepath) 186 parent = os.path.split(path)[0] 187 try: 188 makedirs(parent) 189 except OSError, err: 190 # Reraise error unless if it's an already existing 191 if err.errno != errno.EEXIST or not os.path.isdir(parent): 192 raise 193 data = zipFile.read(filepath) 194 195 # atomically write to path, see #373 196 fd, tempname = tempfile.mkstemp(dir=parent) 197 handle = os.fdopen(fd, 'wb') 198 handle.write(data) 199 handle.close() 200 rename(tempname, path) 201 return directory
202 203
204 -class Bundler:
205 """ 206 I bundle files into a bundle so they can be cached remotely easily. 207 """ 208
209 - def __init__(self, name):
210 """ 211 Create a new bundle. 212 """ 213 self._bundledFiles = {} # dictionary of BundledFile's indexed on path 214 self.name = name 215 self._bundle = Bundle(name)
216
217 - def add(self, source, destination = None):
218 """ 219 Add files to the bundle. 220 221 @param source: the path to the file to add to the bundle. 222 @param destination: a relative path to store this file in the bundle. 223 If unspecified, this will be stored in the top level. 224 225 @returns: the path the file got stored as 226 """ 227 if destination == None: 228 destination = os.path.split(source)[1] 229 self._bundledFiles[source] = BundledFile(source, destination) 230 return destination
231
232 - def bundle(self):
233 """ 234 Bundle the files registered with the bundler. 235 236 @rtype: L{flumotion.common.bundle.Bundle} 237 """ 238 # rescan files registered in the bundle, and check if we need to 239 # rebuild the internal zip 240 if not self._bundle.getZip(): 241 self._bundle.setZip(self._buildzip()) 242 return self._bundle 243 244 update = False 245 for bundledFile in self._bundledFiles.values(): 246 if bundledFile.hasChanged(): 247 update = True 248 break 249 250 if update: 251 self._bundle.setZip(self._buildzip()) 252 253 return self._bundle
254 255 # build the zip file containing the files registered in the bundle 256 # and return the zip file data 257
258 - def _buildzip(self):
259 filelike = StringIO.StringIO() 260 zipFile = zipfile.ZipFile(filelike, "w") 261 for bundledFile in self._bundledFiles.values(): 262 bundledFile.pack(zipFile) 263 zipFile.close() 264 data = filelike.getvalue() 265 filelike.close() 266 return data
267 268
269 -class BundlerBasket:
270 """ 271 I manage bundlers that are registered through me. 272 """ 273
274 - def __init__(self):
275 """ 276 Create a new bundler basket. 277 """ 278 self._bundlers = {} # bundler name -> bundle 279 280 self._files = {} # filename -> bundle name 281 self._imports = {} # import statements -> bundle name 282 283 self._graph = dag.DAG()
284
285 - def add(self, bundleName, source, destination=None):
286 """ 287 Add files to the bundler basket for the given bundle. 288 289 @param bundleName: the name of the bundle this file is a part of 290 @param source: the path to the file to add to the bundle 291 @param destination: a relative path to store this file in the bundle. 292 If unspecified, this will be stored in the top level 293 """ 294 # get the bundler and create it if need be 295 if not bundleName in self._bundlers: 296 bundler = Bundler(bundleName) 297 self._bundlers[bundleName] = bundler 298 else: 299 bundler = self._bundlers[bundleName] 300 301 # add the file to the bundle and register 302 location = bundler.add(source, destination) 303 if location in self._files: 304 raise Exception("Cannot add %s to bundle %s, already in %s" % ( 305 location, bundleName, self._files[location])) 306 self._files[location] = bundleName 307 308 # add possible imports from this file 309 package = None 310 if location.endswith('.py'): 311 package = location[:-3] 312 elif location.endswith('.pyc'): 313 package = location[:-4] 314 315 if package: 316 if package.endswith('__init__'): 317 package = os.path.split(package)[0] 318 319 package = ".".join(package.split('/')) # win32 fixme 320 if package in self._imports: 321 raise Exception("Bundler %s already has import %s" % ( 322 bundleName, package)) 323 self._imports[package] = bundleName
324
325 - def depend(self, depender, *dependencies):
326 """ 327 Make the given bundle depend on the other given bundles. 328 329 @type depender: string 330 @type dependencies: list of strings 331 """ 332 # note that a bundler doesn't necessarily need to be registered yet 333 if not self._graph.hasNode(depender): 334 self._graph.addNode(depender) 335 for dep in dependencies: 336 if not self._graph.hasNode(dep): 337 self._graph.addNode(dep) 338 self._graph.addEdge(depender, dep)
339
340 - def getDependencies(self, bundlerName):
341 """ 342 Return names of all the dependencies of this bundle, including this 343 bundle itself. 344 The dependencies are returned in a correct depending order. 345 """ 346 if not bundlerName in self._bundlers: 347 raise errors.NoBundleError('Unknown bundle %s' % bundlerName) 348 elif not self._graph.hasNode(bundlerName): 349 return [bundlerName] 350 else: 351 return [bundlerName] + self._graph.getOffspring(bundlerName)
352
353 - def getBundlerByName(self, bundlerName):
354 """ 355 Return the bundle by name, or None if not found. 356 """ 357 if bundlerName in self._bundlers: 358 return self._bundlers[bundlerName] 359 return None
360
361 - def getBundlerNameByImport(self, importString):
362 """ 363 Return the bundler name by import statement, or None if not found. 364 """ 365 if importString in self._imports: 366 return self._imports[importString] 367 return None
368
369 - def getBundlerNameByFile(self, filename):
370 """ 371 Return the bundler name by filename, or None if not found. 372 """ 373 if filename in self._files: 374 return self._files[filename] 375 return None
376
377 - def getBundlerNames(self):
378 """ 379 Get all bundler names. 380 381 @rtype: list of str 382 @returns: a list of all bundler names in this basket. 383 """ 384 return self._bundlers.keys()
385 386
387 -class MergedBundler(Bundler):
388 """ 389 I am a bundler, with the extension that I can also bundle other 390 bundlers. 391 392 The effect is that when you call bundle() on a me, you get one 393 bundle with a union of all subbundlers' files, in addition to any 394 loose files that you added to me. 395 """ 396
397 - def __init__(self, name='merged-bundle'):
398 Bundler.__init__(self, name) 399 self._subbundlers = {}
400
401 - def addBundler(self, bundler):
402 """Add to me all of the files managed by another bundler. 403 404 @param bundler: The bundler whose files you want in this 405 bundler. 406 @type bundler: L{Bundler} 407 """ 408 if bundler.name not in self._subbundlers: 409 self._subbundlers[bundler.name] = bundler 410 for bfile in bundler._files.values(): 411 self.add(bfile.source, bfile.destination)
412
413 - def getSubBundlers(self):
414 """ 415 @returns: A list of all of the bundlers that have been added to 416 me. 417 """ 418 return self._subbundlers.values()
419