Package translate :: Package storage :: Module xpi
[hide private]
[frames] | no frames]

Source Code for Module translate.storage.xpi

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  # 
  4  # Copyright 2004, 2005 Zuza Software Foundation 
  5  # 
  6  # This file is part of translate. 
  7  # 
  8  # translate is free software; you can redistribute it and/or modify 
  9  # it under the terms of the GNU General Public License as published by 
 10  # the Free Software Foundation; either version 2 of the License, or 
 11  # (at your option) any later version. 
 12  # 
 13  # translate is distributed in the hope that it will be useful, 
 14  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  # GNU General Public License for more details. 
 17  # 
 18  # You should have received a copy of the GNU General Public License 
 19  # along with translate; if not, write to the Free Software 
 20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 21   
 22  """module for accessing mozilla xpi packages""" 
 23   
 24  from __future__ import generators 
 25  import zipfile 
 26  import os.path 
 27  import StringIO 
 28  import re 
 29   
 30  from translate import __version__ 
 31   
 32  # we have some enhancements to zipfile in a file called zipfileext 
 33  # hopefully they will be included in a future version of python 
 34  from translate.misc import zipfileext 
 35  ZipFileBase = zipfileext.ZipFileExt 
 36   
 37   
 38  from translate.misc import wStringIO 
 39  # this is a fix to the StringIO in Python 2.3.3 
 40  # submitted as patch 951915 on sourceforge 
41 -class FixedStringIO(wStringIO.StringIO):
42
43 - def truncate(self, size=None):
44 StringIO.StringIO.truncate(self, size) 45 self.len = len(self.buf)
46 47 NamedStringInput = wStringIO.StringIO 48 NamedStringOutput = wStringIO.StringIO 49 50
51 -def _commonprefix(itemlist):
52 53 def cp(a, b): 54 l = min(len(a), len(b)) 55 for n in range(l): 56 if a[n] != b[n]: 57 return a[:n] 58 return a[:l]
59 if itemlist: 60 return reduce(cp, itemlist) 61 else: 62 return '' 63 64
65 -def rememberchanged(self, method):
66 67 def changed(*args, **kwargs): 68 self.changed = True 69 method(*args, **kwargs)
70 return changed 71 72
73 -class CatchPotentialOutput(NamedStringInput, object):
74 """catches output if there has been, before closing""" 75
76 - def __init__(self, contents, onclose):
77 """Set up the output stream, and remember a method to call on closing""" 78 NamedStringInput.__init__(self, contents) 79 self.onclose = onclose 80 self.changed = False 81 s = super(CatchPotentialOutput, self) 82 self.write = rememberchanged(self, s.write) 83 self.writelines = rememberchanged(self, s.writelines) 84 self.truncate = rememberchanged(self, s.truncate)
85
86 - def close(self):
87 """wrap the underlying close method, to pass the value to onclose before it goes""" 88 if self.changed: 89 value = self.getvalue() 90 self.onclose(value) 91 NamedStringInput.close(self)
92
93 - def flush(self):
94 """zip files call flush, not close, on file-like objects""" 95 value = self.getvalue() 96 self.onclose(value) 97 NamedStringInput.flush(self)
98
99 - def slam(self):
100 """use this method to force the closing of the stream if it isn't closed yet""" 101 if not self.closed: 102 self.close()
103 104
105 -class ZipFileCatcher(ZipFileBase, object):
106 """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)""" 107
108 - def __init__(self, *args, **kwargs):
109 """initialize the ZipFileCatcher""" 110 # storing oldclose as attribute, since if close is called from __del__ it has no access to external variables 111 self.oldclose = super(ZipFileCatcher, self).close 112 super(ZipFileCatcher, self).__init__(*args, **kwargs)
113
114 - def addcatcher(self, pendingsave):
115 """remember to call the given method before closing""" 116 if hasattr(self, "pendingsaves"): 117 if not pendingsave in self.pendingsaves: 118 self.pendingsaves.append(pendingsave) 119 else: 120 self.pendingsaves = [pendingsave]
121
122 - def close(self):
123 """close the stream, remembering to call any addcatcher methods first""" 124 if hasattr(self, "pendingsaves"): 125 for pendingsave in self.pendingsaves: 126 pendingsave() 127 # if close is called from __del__, it somehow can't see ZipFileCatcher, so we've cached oldclose... 128 if ZipFileCatcher is None: 129 self.oldclose() 130 else: 131 super(ZipFileCatcher, self).close()
132
133 - def overwritestr(self, zinfo_or_arcname, bytes):
134 """writes the string into the archive, overwriting the file if it exists...""" 135 if isinstance(zinfo_or_arcname, zipfile.ZipInfo): 136 filename = zinfo_or_arcname.filename 137 else: 138 filename = zinfo_or_arcname 139 if filename in self.NameToInfo: 140 self.delete(filename) 141 self.writestr(zinfo_or_arcname, bytes) 142 self.writeendrec()
143 144
145 -class XpiFile(ZipFileCatcher):
146
147 - def __init__(self, *args, **kwargs):
148 """sets up the xpi file""" 149 self.includenonloc = kwargs.get("includenonloc", True) 150 if "includenonloc" in kwargs: 151 del kwargs["includenonloc"] 152 if "compression" not in kwargs: 153 kwargs["compression"] = zipfile.ZIP_DEFLATED 154 self.locale = kwargs.pop("locale", None) 155 self.region = kwargs.pop("region", None) 156 super(XpiFile, self).__init__(*args, **kwargs) 157 self.jarfiles = {} 158 self.findlangreg() 159 self.jarprefixes = self.findjarprefixes() 160 self.reverseprefixes = dict([ 161 (prefix, jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix]) 162 self.reverseprefixes["package/"] = None
163
164 - def iterjars(self):
165 """iterate through the jar files in the xpi as ZipFile objects""" 166 for filename in self.namelist(): 167 if filename.lower().endswith('.jar'): 168 if filename not in self.jarfiles: 169 jarstream = self.openinputstream(None, filename) 170 jarfile = ZipFileCatcher(jarstream, mode=self.mode) 171 self.jarfiles[filename] = jarfile 172 else: 173 jarfile = self.jarfiles[filename] 174 yield filename, jarfile
175
176 - def islocfile(self, filename):
177 """returns whether the given file is needed for localization (basically .dtd and .properties)""" 178 base, ext = os.path.splitext(filename) 179 return ext in (os.extsep + "dtd", os.extsep + "properties")
180
181 - def findlangreg(self):
182 """finds the common prefix of all the files stored in the jar files""" 183 dirstructure = {} 184 locale = self.locale 185 region = self.region 186 localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$") 187 regionmatch = re.compile("^[a-zA-Z]{2,3}$") 188 # exclude en-mac, en-win, en-unix for seamonkey 189 osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$") 190 for jarfilename, jarfile in self.iterjars(): 191 jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1) 192 if localematch.match(jarname) and not osmatch.match(jarname): 193 if locale is None: 194 locale = jarname 195 elif locale != jarname: 196 locale = 0 197 elif regionmatch.match(jarname): 198 if region is None: 199 region = jarname 200 elif region != jarname: 201 region = 0 202 for filename in jarfile.namelist(): 203 if filename.endswith('/'): 204 continue 205 if not self.islocfile(filename) and not self.includenonloc: 206 continue 207 parts = filename.split('/')[:-1] 208 treepoint = dirstructure 209 for partnum in range(len(parts)): 210 part = parts[partnum] 211 if part in treepoint: 212 treepoint = treepoint[part] 213 else: 214 treepoint[part] = {} 215 treepoint = treepoint[part] 216 localeentries = {} 217 if 'locale' in dirstructure: 218 for dirname in dirstructure['locale']: 219 localeentries[dirname] = 1 220 if localematch.match(dirname) and not osmatch.match(dirname): 221 if locale is None: 222 locale = dirname 223 elif locale != dirname: 224 print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0" 225 locale = 0 226 elif regionmatch.match(dirname): 227 if region is None: 228 region = dirname 229 elif region != dirname: 230 region = 0 231 if locale and locale in localeentries: 232 del localeentries[locale] 233 if region and region in localeentries: 234 del localeentries[region] 235 if locale and not region: 236 if "-" in locale: 237 region = locale.split("-", 1)[1] 238 else: 239 region = "" 240 self.setlangreg(locale, region)
241
242 - def setlangreg(self, locale, region):
243 """set the locale and region of this xpi""" 244 if locale == 0 or locale is None: 245 raise ValueError("unable to determine locale") 246 self.locale = locale 247 self.region = region 248 self.dirmap = {} 249 if self.locale is not None: 250 self.dirmap[('locale', self.locale)] = ('lang-reg',) 251 if self.region: 252 self.dirmap[('locale', self.region)] = ('reg',)
253
254 - def findjarprefixes(self):
255 """checks the uniqueness of the jar files contents""" 256 uniquenames = {} 257 jarprefixes = {} 258 for jarfilename, jarfile in self.iterjars(): 259 jarprefixes[jarfilename] = "" 260 for filename in jarfile.namelist(): 261 if filename.endswith('/'): 262 continue 263 if filename in uniquenames: 264 jarprefixes[jarfilename] = True 265 jarprefixes[uniquenames[filename]] = True 266 else: 267 uniquenames[filename] = jarfilename 268 for jarfilename, hasconflicts in jarprefixes.items(): 269 if hasconflicts: 270 shortjarfilename = os.path.split(jarfilename)[1] 271 shortjarfilename = os.path.splitext(shortjarfilename)[0] 272 jarprefixes[jarfilename] = shortjarfilename + '/' 273 # this is a clever trick that will e.g. remove zu- from zu-win, zu-mac, zu-unix 274 commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix]) 275 if commonjarprefix: 276 for jarfilename, prefix in jarprefixes.items(): 277 if prefix: 278 jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1) 279 return jarprefixes
280
281 - def ziptoospath(self, zippath):
282 """converts a zipfile filepath to an os-style filepath""" 283 return os.path.join(*zippath.split('/'))
284
285 - def ostozippath(self, ospath):
286 """converts an os-style filepath to a zipfile filepath""" 287 return '/'.join(ospath.split(os.sep))
288
289 - def mapfilename(self, filename):
290 """uses a map to simplify the directory structure""" 291 parts = tuple(filename.split('/')) 292 possiblematch = None 293 for prefix, mapto in self.dirmap.iteritems(): 294 if parts[:len(prefix)] == prefix: 295 if possiblematch is None or len(possiblematch[0]) < len(prefix): 296 possiblematch = prefix, mapto 297 if possiblematch is not None: 298 prefix, mapto = possiblematch 299 mapped = mapto + parts[len(prefix):] 300 return '/'.join(mapped) 301 return filename
302
303 - def mapxpifilename(self, filename):
304 """uses a map to rename files that occur straight in the xpi""" 305 if filename.startswith('bin/chrome/') and filename.endswith(".manifest"): 306 return 'bin/chrome/lang-reg.manifest' 307 return filename
308
309 - def reversemapfile(self, filename):
310 """unmaps the filename...""" 311 possiblematch = None 312 parts = tuple(filename.split('/')) 313 for prefix, mapto in self.dirmap.iteritems(): 314 if parts[:len(mapto)] == mapto: 315 if possiblematch is None or len(possiblematch[0]) < len(mapto): 316 possiblematch = (mapto, prefix) 317 if possiblematch is None: 318 return filename 319 mapto, prefix = possiblematch 320 reversemapped = prefix + parts[len(mapto):] 321 return '/'.join(reversemapped)
322
323 - def reversemapxpifilename(self, filename):
324 """uses a map to rename files that occur straight in the xpi""" 325 if filename == 'bin/chrome/lang-reg.manifest': 326 if self.locale: 327 return '/'.join(('bin', 'chrome', self.locale + '.manifest')) 328 else: 329 for otherfilename in self.namelist(): 330 if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"): 331 return otherfilename 332 return filename
333
334 - def jartoospath(self, jarfilename, filename):
335 """converts a filename from within a jarfile to an os-style filepath""" 336 if jarfilename: 337 jarprefix = self.jarprefixes[jarfilename] 338 return self.ziptoospath(jarprefix + self.mapfilename(filename)) 339 else: 340 return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
341
342 - def ostojarpath(self, ospath):
343 """converts an extracted os-style filepath to a jarfilename and filename""" 344 zipparts = ospath.split(os.sep) 345 prefix = zipparts[0] + '/' 346 if prefix in self.reverseprefixes: 347 jarfilename = self.reverseprefixes[prefix] 348 filename = self.reversemapfile('/'.join(zipparts[1:])) 349 if jarfilename is None: 350 filename = self.reversemapxpifilename(filename) 351 return jarfilename, filename 352 else: 353 filename = self.ostozippath(ospath) 354 if filename in self.namelist(): 355 return None, filename 356 filename = self.reversemapfile('/'.join(zipparts)) 357 possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix] 358 for jarfilename in possiblejarfilenames: 359 jarfile = self.jarfiles[jarfilename] 360 if filename in jarfile.namelist(): 361 return jarfilename, filename 362 raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
363
364 - def jarfileexists(self, jarfilename, filename):
365 """checks whether the given file exists inside the xpi""" 366 if jarfilename is None: 367 return filename in self.namelist() 368 else: 369 jarfile = self.jarfiles[jarfilename] 370 return filename in jarfile.namelist()
371
372 - def ospathexists(self, ospath):
373 """checks whether the given file exists inside the xpi""" 374 jarfilename, filename = self.ostojarpath(ospath) 375 if jarfilename is None: 376 return filename in self.namelist() 377 else: 378 jarfile = self.jarfiles[jarfilename] 379 return filename in jarfile.namelist()
380
381 - def openinputstream(self, jarfilename, filename):
382 """opens a file (possibly inside a jarfile as a StringIO""" 383 if jarfilename is None: 384 contents = self.read(filename) 385 386 def onclose(contents): 387 if contents != self.read(filename): 388 self.overwritestr(filename, contents)
389 inputstream = CatchPotentialOutput(contents, onclose) 390 self.addcatcher(inputstream.slam) 391 else: 392 jarfile = self.jarfiles[jarfilename] 393 contents = jarfile.read(filename) 394 inputstream = NamedStringInput(contents) 395 inputstream.name = self.jartoospath(jarfilename, filename) 396 if hasattr(self.fp, 'name'): 397 inputstream.name = "%s:%s" % (self.fp.name, inputstream.name) 398 return inputstream
399
400 - def openoutputstream(self, jarfilename, filename):
401 """opens a file for writing (possibly inside a jarfile as a StringIO""" 402 if jarfilename is None: 403 404 def onclose(contents): 405 self.overwritestr(filename, contents)
406 else: 407 if jarfilename in self.jarfiles: 408 jarfile = self.jarfiles[jarfilename] 409 else: 410 jarstream = self.openoutputstream(None, jarfilename) 411 jarfile = ZipFileCatcher(jarstream, "w") 412 self.jarfiles[jarfilename] = jarfile 413 self.addcatcher(jarstream.slam) 414 415 def onclose(contents): 416 jarfile.overwritestr(filename, contents) 417 outputstream = wStringIO.CatchStringOutput(onclose) 418 outputstream.name = "%s %s" % (jarfilename, filename) 419 if jarfilename is None: 420 self.addcatcher(outputstream.slam) 421 else: 422 jarfile.addcatcher(outputstream.slam) 423 return outputstream 424
425 - def close(self):
426 """Close the file, and for mode "w" and "a" write the ending records.""" 427 for jarfile in self.jarfiles.itervalues(): 428 jarfile.close() 429 super(XpiFile, self).close()
430
431 - def testzip(self):
432 """test the xpi zipfile and all enclosed jar files...""" 433 for jarfile in self.jarfiles.itervalues(): 434 jarfile.testzip() 435 super(XpiFile, self).testzip()
436
437 - def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
438 """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream""" 439 jarfile = self.jarfiles[origjarfilename] 440 origlang = self.locale[:self.locale.find("-")] 441 if newregion: 442 newlocale = "%s-%s" % (newlang, newregion) 443 else: 444 newlocale = newlang 445 for filename in jarfile.namelist(): 446 filenameparts = filename.split("/") 447 for i in range(len(filenameparts)): 448 part = filenameparts[i] 449 if part == origlang: 450 filenameparts[i] = newlang 451 elif part == self.locale: 452 filenameparts[i] = newlocale 453 elif part == self.region: 454 filenameparts[i] = newregion 455 newfilename = '/'.join(filenameparts) 456 fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename) 457 fileinputstream = self.openinputstream(origjarfilename, filename) 458 fileoutputstream.write(fileinputstream.read()) 459 fileinputstream.close() 460 fileoutputstream.close()
461
462 - def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
463 """Create a new .xpi file with the same contents as this one...""" 464 other = XpiFile(newfilename, "w", locale=newlang, region=newregion) 465 origlang = self.locale[:self.locale.find("-")] 466 # TODO: check if this language replacement code is still neccessary 467 if newlang is None: 468 newlang = origlang 469 if newregion is None: 470 newregion = self.region 471 if newregion: 472 newlocale = "%s-%s" % (newlang, newregion) 473 else: 474 newlocale = newlang 475 for filename in self.namelist(): 476 filenameparts = filename.split('/') 477 basename = filenameparts[-1] 478 if basename.startswith(self.locale): 479 newbasename = basename.replace(self.locale, newlocale) 480 elif basename.startswith(origlang): 481 newbasename = basename.replace(origlang, newlang) 482 elif basename.startswith(self.region): 483 newbasename = basename.replace(self.region, newregion) 484 else: 485 newbasename = basename 486 if newbasename != basename: 487 filenameparts[-1] = newbasename 488 renamefilename = "/".join(filenameparts) 489 print "cloning", filename, "and renaming to", renamefilename 490 else: 491 print "cloning", filename 492 renamefilename = filename 493 if filename.lower().endswith(".jar"): 494 self.restructurejar(filename, renamefilename, other, newlang, newregion) 495 else: 496 inputstream = self.openinputstream(None, filename) 497 outputstream = other.openoutputstream(None, renamefilename) 498 outputstream.write(inputstream.read()) 499 inputstream.close() 500 outputstream.close() 501 other.close() 502 if newmode is None: 503 newmode = self.mode 504 if newmode == "w": 505 newmode = "a" 506 other = XpiFile(newfilename, newmode) 507 other.setlangreg(newlocale, newregion) 508 return other
509
510 - def iterextractnames(self, includenonjars=False, includedirs=False):
511 """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary""" 512 if includenonjars: 513 for filename in self.namelist(): 514 if filename.endswith('/') and not includedirs: 515 continue 516 if not self.islocfile(filename) and not self.includenonloc: 517 continue 518 if not filename.lower().endswith(".jar"): 519 yield self.jartoospath(None, filename) 520 for jarfilename, jarfile in self.iterjars(): 521 for filename in jarfile.namelist(): 522 if filename.endswith('/'): 523 if not includedirs: 524 continue 525 if not self.islocfile(filename) and not self.includenonloc: 526 continue 527 yield self.jartoospath(jarfilename, filename)
528 529 # the following methods are required by translate.convert.ArchiveConvertOptionParser #
530 - def __iter__(self):
531 """iterates through all the files. this is the method use by the converters""" 532 for inputpath in self.iterextractnames(includenonjars=True): 533 yield inputpath
534
535 - def __contains__(self, fullpath):
536 """returns whether the given pathname exists in the archive""" 537 try: 538 jarfilename, filename = self.ostojarpath(fullpath) 539 except IndexError: 540 return False 541 return self.jarfileexists(jarfilename, filename)
542
543 - def openinputfile(self, fullpath):
544 """opens an input file given the full pathname""" 545 jarfilename, filename = self.ostojarpath(fullpath) 546 return self.openinputstream(jarfilename, filename)
547
548 - def openoutputfile(self, fullpath):
549 """opens an output file given the full pathname""" 550 try: 551 jarfilename, filename = self.ostojarpath(fullpath) 552 except IndexError: 553 return None 554 return self.openoutputstream(jarfilename, filename)
555 556 557 if __name__ == '__main__': 558 import optparse 559 optparser = optparse.OptionParser(version="%prog " + __version__.sver) 560 optparser.usage = "%prog [-l|-x] [options] file.xpi" 561 optparser.add_option("-l", "--list", help="list files", \ 562 action="store_true", dest="listfiles", default=False) 563 optparser.add_option("-p", "--prefix", help="show common prefix", \ 564 action="store_true", dest="showprefix", default=False) 565 optparser.add_option("-x", "--extract", help="extract files", \ 566 action="store_true", dest="extractfiles", default=False) 567 optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \ 568 default=".", metavar="EXTRACTDIR") 569 (options, args) = optparser.parse_args() 570 if len(args) < 1: 571 optparser.error("need at least one argument") 572 xpifile = XpiFile(args[0]) 573 if options.showprefix: 574 for prefix, mapto in xpifile.dirmap.iteritems(): 575 print "/".join(prefix), "->", "/".join(mapto) 576 if options.listfiles: 577 for name in xpifile.iterextractnames(includenonjars=True, includedirs=True): 578 print name #, xpifile.ostojarpath(name) 579 if options.extractfiles: 580 if options.extractdir and not os.path.isdir(options.extractdir): 581 os.mkdir(options.extractdir) 582 for name in xpifile.iterextractnames(includenonjars=True, includedirs=False): 583 abspath = os.path.join(options.extractdir, name) 584 # check neccessary directories exist - this way we don't create empty directories 585 currentpath = options.extractdir 586 subparts = os.path.dirname(name).split(os.sep) 587 for part in subparts: 588 currentpath = os.path.join(currentpath, part) 589 if not os.path.isdir(currentpath): 590 os.mkdir(currentpath) 591 outputstream = open(abspath, 'w') 592 jarfilename, filename = xpifile.ostojarpath(name) 593 inputstream = xpifile.openinputstream(jarfilename, filename) 594 outputstream.write(inputstream.read()) 595 outputstream.close() 596