1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 bundles of files used to implement caching over the network
24 """
25
26 import StringIO
27 import errno
28 import os
29 import sys
30 import tempfile
31 import zipfile
32
33 from flumotion.common import errors, dag, python
34 from flumotion.common.python import makedirs
35
36 __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket']
37 __version__ = "$Rev$"
38
39
42
43
45
46
47
48 try:
49 return os.rename(source, dest)
50 except WindowsError, e:
51 import winerror
52 if e.errno == winerror.ERROR_ALREADY_EXISTS:
53 os.unlink(source)
54
55
56 if sys.platform == 'win32':
57 rename = _win32Rename
58
59
61 """
62 I represent one file as managed by a bundler.
63 """
64
65 - def __init__(self, source, destination):
66 self.source = source
67 self.destination = destination
68 self._last_md5sum = None
69 self._last_timestamp = None
70 self.zipped = False
71
73 """
74 Calculate the md5sum of the given file.
75
76 @returns: the md5 sum a 32 character string of hex characters.
77 """
78 data = open(self.source, "r").read()
79 return python.md5(data).hexdigest()
80
82 """
83 @returns: the last modified timestamp for the file.
84 """
85 return os.path.getmtime(self.source)
86
88 """
89 Check if the file has changed since it was last checked.
90
91 @rtype: boolean
92 """
93
94
95
96
97 if not self.zipped:
98 return True
99
100 try:
101 timestamp = self.timestamp()
102 except OSError:
103 return True
104
105
106 if self._last_timestamp and timestamp <= self._last_timestamp:
107 return False
108 self._last_timestamp = timestamp
109
110
111 md5sum = self.md5sum()
112 if self._last_md5sum != md5sum:
113 self._last_md5sum = md5sum
114 return True
115
116 return False
117
118 - def pack(self, zip):
119 self._last_timestamp = self.timestamp()
120 self._last_md5sum = self.md5sum()
121 zip.write(self.source, self.destination)
122 self.zipped = True
123
124
126 """
127 I am a bundle of files, represented by a zip file and md5sum.
128 """
129
134
136 """
137 Set the bundle to the given data representation of the zip file.
138 """
139 self.zip = zip
140 self.md5sum = python.md5(self.zip).hexdigest()
141
143 """
144 Get the bundle's zip data.
145 """
146 return self.zip
147
148
150 """
151 I unbundle bundles by unpacking them in the given directory
152 under directories with the bundle's md5sum.
153 """
154
157
159 """
160 Return the full path where a bundle with the given name and md5sum
161 would be unbundled to.
162 """
163 return os.path.join(self._undir, name, md5sum)
164
170
172 """
173 Unbundle the given bundle.
174
175 @type bundle: L{flumotion.common.bundle.Bundle}
176
177 @rtype: string
178 @returns: the full path to the directory where it was unpacked
179 """
180 directory = self.unbundlePath(bundle)
181
182 filelike = StringIO.StringIO(bundle.getZip())
183 zipFile = zipfile.ZipFile(filelike, "r")
184 zipFile.testzip()
185
186 filepaths = zipFile.namelist()
187 for filepath in filepaths:
188 path = os.path.join(directory, filepath)
189 parent = os.path.split(path)[0]
190 try:
191 makedirs(parent)
192 except OSError, err:
193
194 if err.errno != errno.EEXIST or not os.path.isdir(parent):
195 raise
196 data = zipFile.read(filepath)
197
198
199 fd, tempname = tempfile.mkstemp(dir=parent)
200 handle = os.fdopen(fd, 'wb')
201 handle.write(data)
202 handle.close()
203 rename(tempname, path)
204 return directory
205
206
208 """
209 I bundle files into a bundle so they can be cached remotely easily.
210 """
211
213 """
214 Create a new bundle.
215 """
216 self._bundledFiles = {}
217 self.name = name
218 self._bundle = Bundle(name)
219
220 - def add(self, source, destination = None):
221 """
222 Add files to the bundle.
223
224 @param source: the path to the file to add to the bundle.
225 @param destination: a relative path to store this file in the bundle.
226 If unspecified, this will be stored in the top level.
227
228 @returns: the path the file got stored as
229 """
230 if destination == None:
231 destination = os.path.split(source)[1]
232 self._bundledFiles[source] = BundledFile(source, destination)
233 return destination
234
236 """
237 Bundle the files registered with the bundler.
238
239 @rtype: L{flumotion.common.bundle.Bundle}
240 """
241
242
243 if not self._bundle.getZip():
244 self._bundle.setZip(self._buildzip())
245 return self._bundle
246
247 update = False
248 for bundledFile in self._bundledFiles.values():
249 if bundledFile.hasChanged():
250 update = True
251 break
252
253 if update:
254 self._bundle.setZip(self._buildzip())
255
256 return self._bundle
257
258
259
260
262 filelike = StringIO.StringIO()
263 zipFile = zipfile.ZipFile(filelike, "w")
264 for bundledFile in self._bundledFiles.values():
265 bundledFile.pack(zipFile)
266 zipFile.close()
267 data = filelike.getvalue()
268 filelike.close()
269 return data
270
271
273 """
274 I manage bundlers that are registered through me.
275 """
276
278 """
279 Create a new bundler basket.
280 """
281 self._bundlers = {}
282
283 self._files = {}
284 self._imports = {}
285
286 self._graph = dag.DAG()
287
288 self._mtime = mtime
289
290
292 return self._mtime >= mtime
293
294 - def add(self, bundleName, source, destination=None):
295 """
296 Add files to the bundler basket for the given bundle.
297
298 @param bundleName: the name of the bundle this file is a part of
299 @param source: the path to the file to add to the bundle
300 @param destination: a relative path to store this file in the bundle.
301 If unspecified, this will be stored in the top level
302 """
303
304 if not bundleName in self._bundlers:
305 bundler = Bundler(bundleName)
306 self._bundlers[bundleName] = bundler
307 else:
308 bundler = self._bundlers[bundleName]
309
310
311 location = bundler.add(source, destination)
312 if location in self._files:
313 raise Exception("Cannot add %s to bundle %s, already in %s" % (
314 location, bundleName, self._files[location]))
315 self._files[location] = bundleName
316
317
318 package = None
319 if location.endswith('.py'):
320 package = location[:-3]
321 elif location.endswith('.pyc'):
322 package = location[:-4]
323
324 if package:
325 if package.endswith('__init__'):
326 package = os.path.split(package)[0]
327
328 package = ".".join(package.split('/'))
329 if package in self._imports:
330 raise Exception("Bundler %s already has import %s" % (
331 bundleName, package))
332 self._imports[package] = bundleName
333
334 - def depend(self, depender, *dependencies):
335 """
336 Make the given bundle depend on the other given bundles.
337
338 @type depender: string
339 @type dependencies: list of strings
340 """
341
342 if not self._graph.hasNode(depender):
343 self._graph.addNode(depender)
344 for dep in dependencies:
345 if not self._graph.hasNode(dep):
346 self._graph.addNode(dep)
347 self._graph.addEdge(depender, dep)
348
350 """
351 Return names of all the dependencies of this bundle, including this
352 bundle itself.
353 The dependencies are returned in a correct depending order.
354 """
355 if not bundlerName in self._bundlers:
356 raise errors.NoBundleError('Unknown bundle %s' % bundlerName)
357 elif not self._graph.hasNode(bundlerName):
358 return [bundlerName]
359 else:
360 return [bundlerName] + self._graph.getOffspring(bundlerName)
361
363 """
364 Return the bundle by name, or None if not found.
365 """
366 if bundlerName in self._bundlers:
367 return self._bundlers[bundlerName]
368 return None
369
371 """
372 Return the bundler name by import statement, or None if not found.
373 """
374 if importString in self._imports:
375 return self._imports[importString]
376 return None
377
379 """
380 Return the bundler name by filename, or None if not found.
381 """
382 if filename in self._files:
383 return self._files[filename]
384 return None
385
387 """
388 Get all bundler names.
389
390 @rtype: list of str
391 @returns: a list of all bundler names in this basket.
392 """
393 return self._bundlers.keys()
394
395
397 """
398 I am a bundler, with the extension that I can also bundle other
399 bundlers.
400
401 The effect is that when you call bundle() on a me, you get one
402 bundle with a union of all subbundlers' files, in addition to any
403 loose files that you added to me.
404 """
405
406 - def __init__(self, name='merged-bundle'):
409
411 """Add to me all of the files managed by another bundler.
412
413 @param bundler: The bundler whose files you want in this
414 bundler.
415 @type bundler: L{Bundler}
416 """
417 if bundler.name not in self._subbundlers:
418 self._subbundlers[bundler.name] = bundler
419 for bfile in bundler._files.values():
420 self.add(bfile.source, bfile.destination)
421
423 """
424 @returns: A list of all of the bundlers that have been added to
425 me.
426 """
427 return self._subbundlers.values()
428