FST  openfst-1.8.2
OpenFst Library
fst.cc
Go to the documentation of this file.
1 // Copyright 2005-2020 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the 'License');
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an 'AS IS' BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // See www.openfst.org for extensive documentation on this weighted
16 // finite-state transducer library.
17 //
18 // FST definitions.
19 
20 #include <fst/fst.h>
21 
22 #include <cstdint>
23 #include <sstream>
24 
25 #include <fst/flags.h>
26 #include <fst/log.h>
27 
28 // FST flag definitions.
29 
30 DEFINE_bool(fst_verify_properties, false,
31  "Verify FST properties queried by TestProperties");
32 
33 DEFINE_bool(fst_default_cache_gc, true, "Enable garbage collection of cache");
34 
35 DEFINE_int64(fst_default_cache_gc_limit, 1 << 20LL,
36  "Cache byte size that triggers garbage collection");
37 
38 DEFINE_bool(fst_align, false, "Write FST data aligned where appropriate");
39 
40 DEFINE_string(save_relabel_ipairs, "", "Save input relabel pairs to file");
41 DEFINE_string(save_relabel_opairs, "", "Save output relabel pairs to file");
42 
43 DEFINE_string(fst_read_mode, "read",
44  "Default file reading mode for mappable files");
45 
46 namespace fst {
47 
48 // Checks FST magic number and reads in the header; if rewind = true,
49 // the stream is repositioned before call if possible.
50 bool FstHeader::Read(std::istream &strm, const std::string &source,
51  bool rewind) {
52  int64_t pos = 0;
53  if (rewind) pos = strm.tellg();
54  int32_t magic_number = 0;
55  ReadType(strm, &magic_number);
56  if (magic_number != kFstMagicNumber) {
57  LOG(ERROR) << "FstHeader::Read: Bad FST header: " << source
58  << ". Magic number not matched. Got: " << magic_number;
59  if (rewind) strm.seekg(pos);
60  return false;
61  }
62  ReadType(strm, &fsttype_);
63  ReadType(strm, &arctype_);
64  ReadType(strm, &version_);
65  ReadType(strm, &flags_);
66  ReadType(strm, &properties_);
67  ReadType(strm, &start_);
68  ReadType(strm, &numstates_);
69  ReadType(strm, &numarcs_);
70  if (!strm) {
71  LOG(ERROR) << "FstHeader::Read: Read failed: " << source;
72  return false;
73  }
74  if (rewind) strm.seekg(pos);
75  return true;
76 }
77 
78 // Writes FST magic number and FST header.
79 bool FstHeader::Write(std::ostream &strm, const std::string &) const {
81  WriteType(strm, fsttype_);
82  WriteType(strm, arctype_);
83  WriteType(strm, version_);
84  WriteType(strm, flags_);
85  WriteType(strm, properties_);
86  WriteType(strm, start_);
87  WriteType(strm, numstates_);
88  WriteType(strm, numarcs_);
89  return true;
90 }
91 
92 std::string FstHeader::DebugString() const {
93  std::ostringstream ostrm;
94  ostrm << "fsttype: \"" << fsttype_ << "\" arctype: \"" << arctype_
95  << "\" version: \"" << version_ << "\" flags: \"" << flags_
96  << "\" properties: \"" << properties_ << "\" start: \"" << start_
97  << "\" numstates: \"" << numstates_ << "\" numarcs: \"" << numarcs_
98  << "\"";
99  return ostrm.str();
100 }
101 
102 FstReadOptions::FstReadOptions(const std::string_view source,
103  const FstHeader *header,
104  const SymbolTable *isymbols,
105  const SymbolTable *osymbols)
106  : source(source),
107  header(header),
108  isymbols(isymbols),
109  osymbols(osymbols),
110  read_isymbols(true),
111  read_osymbols(true) {
112  mode = ReadMode(FST_FLAGS_fst_read_mode);
113 }
114 
115 FstReadOptions::FstReadOptions(const std::string_view source,
116  const SymbolTable *isymbols,
117  const SymbolTable *osymbols)
118  : FstReadOptions(source, /*header=*/nullptr, isymbols, osymbols) {}
119 
121  if (mode == "read") return READ;
122  if (mode == "map") return MAP;
123  LOG(ERROR) << "Unknown file read mode " << mode;
124  return READ;
125 }
126 
127 std::string FstReadOptions::DebugString() const {
128  std::ostringstream ostrm;
129  ostrm << "source: \"" << source << "\" mode: \""
130  << (mode == READ ? "READ" : "MAP") << "\" read_isymbols: \""
131  << (read_isymbols ? "true" : "false") << "\" read_osymbols: \""
132  << (read_osymbols ? "true" : "false") << "\" header: \""
133  << (header ? "set" : "null") << "\" isymbols: \""
134  << (isymbols ? "set" : "null") << "\" osymbols: \""
135  << (osymbols ? "set" : "null") << "\"";
136  return ostrm.str();
137 }
138 
139 } // namespace fst
bool Read(std::istream &strm, const std::string &source, bool rewind=false)
Definition: fst.cc:50
constexpr int32_t kFstMagicNumber
Definition: fst.h:55
const FstHeader * header
Definition: fst.h:75
std::string source
Definition: fst.h:74
#define LOG(type)
Definition: log.h:49
bool read_isymbols
Definition: fst.h:82
const SymbolTable * osymbols
Definition: fst.h:79
FstReadOptions(const std::string_view source="<unspecified>", const FstHeader *header=nullptr, const SymbolTable *isymbols=nullptr, const SymbolTable *osymbols=nullptr)
Definition: fst.cc:102
std::string DebugString() const
Definition: fst.cc:92
std::ostream & WriteType(std::ostream &strm, const T t)
Definition: util.h:211
std::string DebugString() const
Definition: fst.cc:127
static FileReadMode ReadMode(std::string_view mode)
Definition: fst.cc:120
DEFINE_string(save_relabel_ipairs,"","Save input relabel pairs to file")
DEFINE_int64(fst_default_cache_gc_limit, 1<< 20LL,"Cache byte size that triggers garbage collection")
bool Write(std::ostream &strm, const std::string &source) const
Definition: fst.cc:79
FileReadMode mode
Definition: fst.h:81
std::istream & ReadType(std::istream &strm, T *t)
Definition: util.h:65
const SymbolTable * isymbols
Definition: fst.h:77
DEFINE_bool(fst_verify_properties, false,"Verify FST properties queried by TestProperties")
bool read_osymbols
Definition: fst.h:83