FST  openfst-1.7.1
OpenFst Library
mapped-file.cc
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 
5 #include <fst/mapped-file.h>
6 
7 #include <errno.h>
8 #include <fcntl.h>
9 #include <sys/mman.h>
10 #include <sys/types.h>
11 #include <unistd.h>
12 
13 #include <algorithm>
14 #include <ios>
15 #include <memory>
16 
17 #include <fst/log.h>
18 
19 namespace fst {
20 
21 MappedFile::MappedFile(const MemoryRegion &region) : region_(region) {}
22 
24  if (region_.size != 0) {
25  if (region_.mmap) {
26  VLOG(1) << "munmap'ed " << region_.size << " bytes at " << region_.mmap;
27  if (munmap(region_.mmap, region_.size) != 0) {
28  LOG(ERROR) << "Failed to unmap region: " << strerror(errno);
29  }
30  } else {
31  if (region_.data) {
32  operator delete(static_cast<char *>(region_.data) - region_.offset);
33  }
34  }
35  }
36 }
37 
38 MappedFile *MappedFile::Map(std::istream *istrm, bool memorymap,
39  const string &source, size_t size) {
40  const auto spos = istrm->tellg();
41  VLOG(1) << "memorymap: " << (memorymap ? "true" : "false") << " source: \""
42  << source << "\""
43  << " size: " << size << " offset: " << spos;
44  if (memorymap && spos >= 0 && spos % kArchAlignment == 0) {
45  const size_t pos = spos;
46  const int fd = open(source.c_str(), O_RDONLY);
47  if (fd != -1) {
48  std::unique_ptr<MappedFile> mmf(MapFromFileDescriptor(fd, pos, size));
49  if (close(fd) == 0 && mmf != nullptr) {
50  istrm->seekg(pos + size, std::ios::beg);
51  if (istrm) {
52  VLOG(1) << "mmap'ed region of " << size << " at offset " << pos
53  << " from " << source << " to addr " << mmf->region_.mmap;
54  return mmf.release();
55  }
56  } else {
57  LOG(INFO) << "Mapping of file failed: " << strerror(errno);
58  }
59  }
60  }
61 
62  // If all else fails, reads from the file into the allocated buffer.
63  if (memorymap) {
64  LOG(WARNING) << "File mapping at offset " << spos << " of file " << source
65  << " could not be honored, reading instead";
66  }
67  // Reads the file into the buffer in chunks not larger than kMaxReadChunk.
68  std::unique_ptr<MappedFile> mf(Allocate(size));
69  auto *buffer = reinterpret_cast<char *>(mf->mutable_data());
70  while (size > 0) {
71  const auto next_size = std::min(size, kMaxReadChunk);
72  const auto current_pos = istrm->tellg();
73  if (!istrm->read(buffer, next_size)) {
74  LOG(ERROR) << "Failed to read " << next_size << " bytes at offset "
75  << current_pos << "from \"" << source << "\"";
76  return nullptr;
77  }
78  size -= next_size;
79  buffer += next_size;
80  VLOG(2) << "Read " << next_size << " bytes. " << size << " remaining";
81  }
82  return mf.release();
83 }
84 
85 MappedFile *MappedFile::MapFromFileDescriptor(int fd, int pos, size_t size) {
86  const int pagesize = sysconf(_SC_PAGESIZE);
87  const off_t offset = pos % pagesize;
88  const off_t upsize = size + offset;
89  void *map = mmap(nullptr, upsize, PROT_READ, MAP_SHARED, fd, pos - offset);
90  if (map == MAP_FAILED) {
91  LOG(ERROR) << "mmap failed for fd=" << fd << " size=" << upsize
92  << " offset=" << (pos - offset);
93  return nullptr;
94  }
95 
96  MemoryRegion region;
97  region.mmap = map;
98  region.size = upsize;
99  region.data =
100  reinterpret_cast<void *>(reinterpret_cast<char *>(map) + offset);
101  region.offset = offset;
102  return new MappedFile(region);
103 }
104 
105 MappedFile *MappedFile::Allocate(size_t size, int align) {
106  MemoryRegion region;
107  region.data = nullptr;
108  region.offset = 0;
109  if (size > 0) {
110  char *buffer = static_cast<char *>(operator new(size + align));
111  size_t address = reinterpret_cast<size_t>(buffer);
112  region.offset = kArchAlignment - (address % align);
113  region.data = buffer + region.offset;
114  }
115  region.mmap = nullptr;
116  region.size = size;
117  return new MappedFile(region);
118 }
119 
121  MemoryRegion region;
122  region.data = data;
123  region.mmap = data;
124  region.size = 0;
125  region.offset = 0;
126  return new MappedFile(region);
127 }
128 
129 constexpr int MappedFile::kArchAlignment;
130 
131 constexpr size_t MappedFile::kMaxReadChunk;
132 
133 } // namespace fst
#define LOG(type)
Definition: log.h:48
static constexpr int kArchAlignment
Definition: mapped-file.h:68
static constexpr size_t kMaxReadChunk
Definition: mapped-file.h:70
#define VLOG(level)
Definition: log.h:49
static MappedFile * Map(std::istream *istrm, bool memorymap, const string &source, size_t size)
Definition: mapped-file.cc:38
string source
Definition: fst.h:57
static MappedFile * Borrow(void *data)
Definition: mapped-file.cc:120
static MappedFile * Allocate(size_t size, int align=kArchAlignment)
Definition: mapped-file.cc:105
static MappedFile * MapFromFileDescriptor(int fd, int pos, size_t size)
Definition: mapped-file.cc:85