Libosmium  2.15.6
Fast and flexible C++ library for working with OpenStreetMap data
bzip2_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_BZIP2_COMPRESSION_HPP
2 #define OSMIUM_IO_BZIP2_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2020 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
50 #include <osmium/util/file.hpp>
51 
52 #include <bzlib.h>
53 
54 #include <cassert>
55 #include <cerrno>
56 #include <cstdio>
57 #include <limits>
58 #include <string>
59 #include <system_error>
60 
61 #ifndef _MSC_VER
62 # include <unistd.h>
63 #endif
64 
65 namespace osmium {
66 
71  struct bzip2_error : public io_error {
72 
74  int system_errno = 0;
75 
76  bzip2_error(const std::string& what, const int error_code) :
77  io_error(what),
78  bzip2_error_code(error_code) {
79  if (error_code == BZ_IO_ERROR) {
80  system_errno = errno;
81  }
82  }
83 
84  }; // struct bzip2_error
85 
86  namespace io {
87 
88  namespace detail {
89 
90  [[noreturn]] inline void throw_bzip2_error(BZFILE* bzfile, const char* msg, const int bzlib_error) {
91  std::string error{"bzip2 error: "};
92  error += msg;
93  error += ": ";
94  int errnum = bzlib_error;
95  if (bzlib_error) {
96  error += std::to_string(bzlib_error);
97  } else if (bzfile) {
98  error += ::BZ2_bzerror(bzfile, &errnum);
99  }
100  throw osmium::bzip2_error{error, errnum};
101  }
102 
103  class file_wrapper {
104 
105  FILE* m_file = nullptr;
106 
107  public:
108 
109  file_wrapper() noexcept = default;
110 
111  file_wrapper(const int fd, const char* mode) {
112 #ifdef _MSC_VER
113  osmium::detail::disable_invalid_parameter_handler diph;
114 #endif
115  m_file = fdopen(fd, mode);
116  if (!m_file) {
117 
118  // Do not close stdout
119  if (fd != 1) {
120  ::close(fd);
121  }
122  throw std::system_error{errno, std::system_category(), "fdopen failed"};
123  }
124  }
125 
126  file_wrapper(const file_wrapper&) = delete;
127  file_wrapper& operator=(const file_wrapper&) = delete;
128 
129  file_wrapper(file_wrapper&&) = delete;
130  file_wrapper& operator=(file_wrapper&&) = delete;
131 
132  ~file_wrapper() noexcept {
133 #ifdef _MSC_VER
134  osmium::detail::disable_invalid_parameter_handler diph;
135 #endif
136  if (m_file) {
137  fclose(m_file);
138  }
139  }
140 
141  FILE* file() const noexcept {
142  return m_file;
143  }
144 
145  void close() {
146 #ifdef _MSC_VER
147  osmium::detail::disable_invalid_parameter_handler diph;
148 #endif
149  if (m_file) {
150  FILE* file = m_file;
151  m_file = nullptr;
152 
153  // Do not close stdout
154  if (fileno(file) == 1) {
155  return;
156  }
157 
158  if (fclose(file) != 0) {
159  throw std::system_error{errno, std::system_category(), "fclose failed"};
160  }
161  }
162  }
163 
164  }; // class file_wrapper
165 
166  } // namespace detail
167 
168  class Bzip2Compressor final : public Compressor {
169 
170  detail::file_wrapper m_file;
171  BZFILE* m_bzfile = nullptr;
172 
173  public:
174 
175  explicit Bzip2Compressor(const int fd, const fsync sync) :
176  Compressor(sync),
177  m_file(fd, "wb") {
178 #ifdef _MSC_VER
179  osmium::detail::disable_invalid_parameter_handler diph;
180 #endif
181  int bzerror = BZ_OK;
182  m_bzfile = ::BZ2_bzWriteOpen(&bzerror, m_file.file(), 6, 0, 0);
183  if (!m_bzfile) {
184  throw bzip2_error{"bzip2 error: write open failed", bzerror};
185  }
186  }
187 
190 
193 
194  ~Bzip2Compressor() noexcept {
195  try {
196  close();
197  } catch (...) {
198  // Ignore any exceptions because destructor must not throw.
199  }
200  }
201 
202  void write(const std::string& data) override {
203  assert(data.size() < std::numeric_limits<int>::max());
204  assert(m_bzfile);
205 #ifdef _MSC_VER
206  osmium::detail::disable_invalid_parameter_handler diph;
207 #endif
208  int bzerror = BZ_OK;
209  ::BZ2_bzWrite(&bzerror, m_bzfile, const_cast<char*>(data.data()), static_cast<int>(data.size()));
210  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
211  detail::throw_bzip2_error(m_bzfile, "write failed", bzerror);
212  }
213  }
214 
215  void close() override {
216  if (m_bzfile) {
217 #ifdef _MSC_VER
218  osmium::detail::disable_invalid_parameter_handler diph;
219 #endif
220  int bzerror = BZ_OK;
221  ::BZ2_bzWriteClose(&bzerror, m_bzfile, 0, nullptr, nullptr);
222  m_bzfile = nullptr;
223  if (do_fsync() && m_file.file()) {
224  osmium::io::detail::reliable_fsync(fileno(m_file.file()));
225  }
226  m_file.close();
227  if (bzerror != BZ_OK) {
228  throw bzip2_error{"bzip2 error: write close failed", bzerror};
229  }
230  }
231  }
232 
233  }; // class Bzip2Compressor
234 
235  class Bzip2Decompressor final : public Decompressor {
236 
237  detail::file_wrapper m_file;
238  BZFILE* m_bzfile = nullptr;
239  bool m_stream_end = false;
240 
241  public:
242 
243  explicit Bzip2Decompressor(const int fd) :
244  m_file(fd, "rb") {
245 #ifdef _MSC_VER
246  osmium::detail::disable_invalid_parameter_handler diph;
247 #endif
248  int bzerror = BZ_OK;
249  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, nullptr, 0);
250  if (!m_bzfile) {
251  throw bzip2_error{"bzip2 error: read open failed", bzerror};
252  }
253  }
254 
257 
260 
261  ~Bzip2Decompressor() noexcept {
262  try {
263  close();
264  } catch (...) {
265  // Ignore any exceptions because destructor must not throw.
266  }
267  }
268 
269  std::string read() override {
270 #ifdef _MSC_VER
271  osmium::detail::disable_invalid_parameter_handler diph;
272 #endif
273  assert(m_bzfile);
274  std::string buffer;
275 
276  if (!m_stream_end) {
278  int bzerror = BZ_OK;
279  assert(buffer.size() < std::numeric_limits<int>::max());
280  const int nread = ::BZ2_bzRead(&bzerror, m_bzfile, &*buffer.begin(), static_cast<int>(buffer.size()));
281  if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) {
282  detail::throw_bzip2_error(m_bzfile, "read failed", bzerror);
283  }
284  if (bzerror == BZ_STREAM_END) {
285  void* unused;
286  int nunused;
287  if (!feof(m_file.file())) {
288  ::BZ2_bzReadGetUnused(&bzerror, m_bzfile, &unused, &nunused);
289  if (bzerror != BZ_OK) {
290  detail::throw_bzip2_error(m_bzfile, "get unused failed", bzerror);
291  }
292  std::string unused_data{static_cast<const char*>(unused), static_cast<std::string::size_type>(nunused)};
293  ::BZ2_bzReadClose(&bzerror, m_bzfile);
294  if (bzerror != BZ_OK) {
295  throw bzip2_error{"bzip2 error: read close failed", bzerror};
296  }
297  assert(unused_data.size() < std::numeric_limits<int>::max());
298  m_bzfile = ::BZ2_bzReadOpen(&bzerror, m_file.file(), 0, 0, &*unused_data.begin(), static_cast<int>(unused_data.size()));
299  if (!m_bzfile) {
300  throw bzip2_error{"bzip2 error: read open failed", bzerror};
301  }
302  } else {
303  m_stream_end = true;
304  }
305  }
306  buffer.resize(static_cast<std::string::size_type>(nread));
307  }
308 
309  set_offset(static_cast<std::size_t>(ftell(m_file.file())));
310 
311  return buffer;
312  }
313 
314  void close() override {
315  if (m_bzfile) {
316 #ifdef _MSC_VER
317  osmium::detail::disable_invalid_parameter_handler diph;
318 #endif
319  int bzerror = BZ_OK;
320  ::BZ2_bzReadClose(&bzerror, m_bzfile);
321  m_bzfile = nullptr;
322  m_file.close();
323  if (bzerror != BZ_OK) {
324  throw bzip2_error{"bzip2 error: read close failed", bzerror};
325  }
326  }
327  }
328 
329  }; // class Bzip2Decompressor
330 
331  class Bzip2BufferDecompressor final : public Decompressor {
332 
333  const char* m_buffer;
334  std::size_t m_buffer_size;
335  bz_stream m_bzstream;
336 
337  public:
338 
339  Bzip2BufferDecompressor(const char* buffer, const std::size_t size) :
340  m_buffer(buffer),
341  m_buffer_size(size),
342  m_bzstream() {
343  m_bzstream.next_in = const_cast<char*>(buffer);
344  assert(size < std::numeric_limits<unsigned int>::max());
345  m_bzstream.avail_in = static_cast<unsigned int>(size);
346  const int result = BZ2_bzDecompressInit(&m_bzstream, 0, 0);
347  if (result != BZ_OK) {
348  throw bzip2_error{"bzip2 error: decompression init failed: ", result};
349  }
350  }
351 
354 
357 
359  try {
360  close();
361  } catch (...) {
362  // Ignore any exceptions because destructor must not throw.
363  }
364  }
365 
366  std::string read() override {
367  std::string output;
368 
369  if (m_buffer) {
370  const std::size_t buffer_size = 10240;
371  output.resize(buffer_size);
372  m_bzstream.next_out = &*output.begin();
373  m_bzstream.avail_out = buffer_size;
374  const int result = BZ2_bzDecompress(&m_bzstream);
375 
376  if (result != BZ_OK) {
377  m_buffer = nullptr;
378  m_buffer_size = 0;
379  }
380 
381  if (result != BZ_OK && result != BZ_STREAM_END) {
382  throw bzip2_error{"bzip2 error: decompress failed: ", result};
383  }
384 
385  output.resize(static_cast<std::size_t>(m_bzstream.next_out - output.data()));
386  }
387 
388  return output;
389  }
390 
391  void close() override {
392  BZ2_bzDecompressEnd(&m_bzstream);
393  }
394 
395  }; // class Bzip2BufferDecompressor
396 
397  namespace detail {
398 
399  // we want the register_compression() function to run, setting
400  // the variable is only a side-effect, it will never be used
402  [](const int fd, const fsync sync) { return new osmium::io::Bzip2Compressor{fd, sync}; },
403  [](const int fd) { return new osmium::io::Bzip2Decompressor{fd}; },
404  [](const char* buffer, const std::size_t size) { return new osmium::io::Bzip2BufferDecompressor{buffer, size}; }
405  );
406 
407  // dummy function to silence the unused variable warning from above
408  inline bool get_registered_bzip2_compression() noexcept {
409  return registered_bzip2_compression;
410  }
411 
412  } // namespace detail
413 
414  } // namespace io
415 
416 } // namespace osmium
417 
418 #endif // OSMIUM_IO_BZIP2_COMPRESSION_HPP
osmium::io::Bzip2Decompressor::Bzip2Decompressor
Bzip2Decompressor(const int fd)
Definition: bzip2_compression.hpp:243
osmium::io::Bzip2Decompressor::operator=
Bzip2Decompressor & operator=(Bzip2Decompressor &&)=delete
osmium::io::Bzip2Compressor::Bzip2Compressor
Bzip2Compressor(const Bzip2Compressor &)=delete
writer_options.hpp
osmium::io::Bzip2Decompressor::close
void close() override
Definition: bzip2_compression.hpp:314
osmium::io::Bzip2Decompressor::m_bzfile
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:238
osmium::io::Bzip2Compressor::Bzip2Compressor
Bzip2Compressor(const int fd, const fsync sync)
Definition: bzip2_compression.hpp:175
osmium::io::CompressionFactory::instance
static CompressionFactory & instance()
Definition: compression.hpp:180
osmium::io::Bzip2Decompressor::m_stream_end
bool m_stream_end
Definition: bzip2_compression.hpp:239
osmium::io::Bzip2BufferDecompressor::Bzip2BufferDecompressor
Bzip2BufferDecompressor(Bzip2BufferDecompressor &&)=delete
osmium::io::Bzip2Decompressor
Definition: bzip2_compression.hpp:235
compression.hpp
osmium::io::Bzip2Decompressor::m_file
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:237
osmium::bzip2_error
Definition: bzip2_compression.hpp:71
detail
Definition: attr.hpp:342
osmium::bzip2_error::system_errno
int system_errno
Definition: bzip2_compression.hpp:74
osmium::io::Compressor
Definition: compression.hpp:57
osmium::io::Bzip2Compressor::m_file
detail::file_wrapper m_file
Definition: bzip2_compression.hpp:170
osmium::io::Decompressor
Definition: compression.hpp:87
osmium::io::Bzip2BufferDecompressor::operator=
Bzip2BufferDecompressor & operator=(Bzip2BufferDecompressor &&)=delete
osmium::io::Bzip2Compressor::m_bzfile
BZFILE * m_bzfile
Definition: bzip2_compression.hpp:171
osmium::io::Bzip2BufferDecompressor::operator=
Bzip2BufferDecompressor & operator=(const Bzip2BufferDecompressor &)=delete
osmium::io::Bzip2Decompressor::read
std::string read() override
Definition: bzip2_compression.hpp:269
osmium::io::Bzip2BufferDecompressor::Bzip2BufferDecompressor
Bzip2BufferDecompressor(const Bzip2BufferDecompressor &)=delete
osmium::io::Bzip2Compressor::close
void close() override
Definition: bzip2_compression.hpp:215
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
osmium::io::Bzip2Compressor::Bzip2Compressor
Bzip2Compressor(Bzip2Compressor &&)=delete
osmium::bzip2_error::bzip2_error
bzip2_error(const std::string &what, const int error_code)
Definition: bzip2_compression.hpp:76
osmium::io::Decompressor::input_buffer_size
@ input_buffer_size
Definition: compression.hpp:95
osmium::io::Bzip2Compressor::operator=
Bzip2Compressor & operator=(Bzip2Compressor &&)=delete
osmium::io_error
Definition: error.hpp:44
osmium::io::Bzip2BufferDecompressor::Bzip2BufferDecompressor
Bzip2BufferDecompressor(const char *buffer, const std::size_t size)
Definition: bzip2_compression.hpp:339
osmium::io::Bzip2BufferDecompressor::close
void close() override
Definition: bzip2_compression.hpp:391
osmium::io::Bzip2BufferDecompressor
Definition: bzip2_compression.hpp:331
osmium::io::Bzip2BufferDecompressor::~Bzip2BufferDecompressor
~Bzip2BufferDecompressor() noexcept
Definition: bzip2_compression.hpp:358
osmium::io::file_compression::bzip2
@ bzip2
osmium::io::fsync
fsync
Definition: writer_options.hpp:51
osmium::io::Bzip2BufferDecompressor::m_buffer_size
std::size_t m_buffer_size
Definition: bzip2_compression.hpp:334
osmium::io::Bzip2Decompressor::Bzip2Decompressor
Bzip2Decompressor(Bzip2Decompressor &&)=delete
osmium::io::Bzip2BufferDecompressor::m_bzstream
bz_stream m_bzstream
Definition: bzip2_compression.hpp:335
osmium::io::Bzip2BufferDecompressor::m_buffer
const char * m_buffer
Definition: bzip2_compression.hpp:333
osmium::bzip2_error::bzip2_error_code
int bzip2_error_code
Definition: bzip2_compression.hpp:73
file.hpp
file_compression.hpp
osmium::io::Decompressor::set_offset
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:124
osmium::io::CompressionFactory::register_compression
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:185
osmium::io::Bzip2BufferDecompressor::read
std::string read() override
Definition: bzip2_compression.hpp:366
osmium::io::Bzip2Decompressor::~Bzip2Decompressor
~Bzip2Decompressor() noexcept
Definition: bzip2_compression.hpp:261
osmium::io::Bzip2Compressor::~Bzip2Compressor
~Bzip2Compressor() noexcept
Definition: bzip2_compression.hpp:194
osmium::io::Compressor::do_fsync
bool do_fsync() const noexcept
Definition: compression.hpp:63
osmium::io::Bzip2Decompressor::Bzip2Decompressor
Bzip2Decompressor(const Bzip2Decompressor &)=delete
osmium::io::Bzip2Compressor::write
void write(const std::string &data) override
Definition: bzip2_compression.hpp:202
osmium::io::Bzip2Compressor::operator=
Bzip2Compressor & operator=(const Bzip2Compressor &)=delete
osmium::io::Bzip2Decompressor::operator=
Bzip2Decompressor & operator=(const Bzip2Decompressor &)=delete
error.hpp
osmium::io::Bzip2Compressor
Definition: bzip2_compression.hpp:168