FST  openfst-1.7.1
OpenFst Library
fst.cc
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 // FST definitions.
5 
6 #include <fst/fst.h>
7 
8 #include <sstream>
9 
10 #include <fst/flags.h>
11 #include <fst/log.h>
12 #include <fst/matcher-fst.h> // declarations of *_lookahead_fst_type
13 
14 // FST flag definitions.
15 
16 DEFINE_bool(fst_verify_properties, false,
17  "Verify FST properties queried by TestProperties");
18 
19 DEFINE_bool(fst_default_cache_gc, true, "Enable garbage collection of cache");
20 
21 DEFINE_int64(fst_default_cache_gc_limit, 1 << 20LL,
22  "Cache byte size that triggers garbage collection");
23 
24 DEFINE_bool(fst_align, false, "Write FST data aligned where appropriate");
25 
26 DEFINE_string(save_relabel_ipairs, "", "Save input relabel pairs to file");
27 DEFINE_string(save_relabel_opairs, "", "Save output relabel pairs to file");
28 
29 DEFINE_string(fst_read_mode, "read",
30  "Default file reading mode for mappable files");
31 
32 namespace fst {
33 
34 // FST type definitions for lookahead FSTs.
35 const char arc_lookahead_fst_type[] = "arc_lookahead";
36 const char ilabel_lookahead_fst_type[] = "ilabel_lookahead";
37 const char olabel_lookahead_fst_type[] = "olabel_lookahead";
38 
39 // Identifies stream data as an FST (and its endianity).
40 constexpr int32 kFstMagicNumber = 2125659606;
41 
42 // Checks for FST magic number in stream, to indicate caller function that the
43 // stream content is an FST header.
44 bool IsFstHeader(std::istream &strm, const string &source) {
45  int64 pos = strm.tellg();
46  bool match = true;
47  int32 magic_number = 0;
48  ReadType(strm, &magic_number);
49  if (magic_number != kFstMagicNumber) {
50  match = false;
51  }
52  strm.seekg(pos);
53  return match;
54 }
55 
56 // Checks FST magic number and reads in the header; if rewind = true,
57 // the stream is repositioned before call if possible.
58 bool FstHeader::Read(std::istream &strm, const string &source, bool rewind) {
59  int64 pos = 0;
60  if (rewind) pos = strm.tellg();
61  int32 magic_number = 0;
62  ReadType(strm, &magic_number);
63  if (magic_number != kFstMagicNumber) {
64  LOG(ERROR) << "FstHeader::Read: Bad FST header: " << source;
65  if (rewind) strm.seekg(pos);
66  return false;
67  }
68  ReadType(strm, &fsttype_);
69  ReadType(strm, &arctype_);
70  ReadType(strm, &version_);
71  ReadType(strm, &flags_);
72  ReadType(strm, &properties_);
73  ReadType(strm, &start_);
74  ReadType(strm, &numstates_);
75  ReadType(strm, &numarcs_);
76  if (!strm) {
77  LOG(ERROR) << "FstHeader::Read: Read failed: " << source;
78  return false;
79  }
80  if (rewind) strm.seekg(pos);
81  return true;
82 }
83 
84 // Writes FST magic number and FST header.
85 bool FstHeader::Write(std::ostream &strm, const string &source) const {
86  WriteType(strm, kFstMagicNumber);
87  WriteType(strm, fsttype_);
88  WriteType(strm, arctype_);
89  WriteType(strm, version_);
90  WriteType(strm, flags_);
91  WriteType(strm, properties_);
92  WriteType(strm, start_);
93  WriteType(strm, numstates_);
94  WriteType(strm, numarcs_);
95  return true;
96 }
97 
98 string FstHeader::DebugString() const {
99  std::ostringstream ostrm;
100  ostrm << "fsttype: \"" << fsttype_ << "\" arctype: \"" << arctype_
101  << "\" version: \"" << version_ << "\" flags: \"" << flags_
102  << "\" properties: \"" << properties_ << "\" start: \"" << start_
103  << "\" numstates: \"" << numstates_ << "\" numarcs: \"" << numarcs_
104  << "\"";
105  return ostrm.str();
106 }
107 
108 FstReadOptions::FstReadOptions(const string &source, const FstHeader *header,
109  const SymbolTable *isymbols,
110  const SymbolTable *osymbols)
111  : source(source),
112  header(header),
113  isymbols(isymbols),
114  osymbols(osymbols),
115  read_isymbols(true),
116  read_osymbols(true) {
117  mode = ReadMode(FLAGS_fst_read_mode);
118 }
119 
121  const SymbolTable *isymbols,
122  const SymbolTable *osymbols)
123  : source(source),
124  header(nullptr),
125  isymbols(isymbols),
126  osymbols(osymbols),
127  read_isymbols(true),
128  read_osymbols(true) {
129  mode = ReadMode(FLAGS_fst_read_mode);
130 }
131 
133  if (mode == "read") return READ;
134  if (mode == "map") return MAP;
135  LOG(ERROR) << "Unknown file read mode " << mode;
136  return READ;
137 }
138 
140  std::ostringstream ostrm;
141  ostrm << "source: \"" << source << "\" mode: \""
142  << (mode == READ ? "READ" : "MAP") << "\" read_isymbols: \""
143  << (read_isymbols ? "true" : "false") << "\" read_osymbols: \""
144  << (read_osymbols ? "true" : "false") << "\" header: \""
145  << (header ? "set" : "null") << "\" isymbols: \""
146  << (isymbols ? "set" : "null") << "\" osymbols: \""
147  << (osymbols ? "set" : "null") << "\"";
148  return ostrm.str();
149 }
150 
151 } // namespace fst
const char ilabel_lookahead_fst_type[]
Definition: fst.cc:36
const FstHeader * header
Definition: fst.h:58
const char olabel_lookahead_fst_type[]
Definition: fst.cc:37
const char arc_lookahead_fst_type[]
Definition: fst.cc:35
bool IsFstHeader(std::istream &, const string &)
Definition: fst.cc:44
#define LOG(type)
Definition: log.h:48
bool read_isymbols
Definition: fst.h:65
string DebugString() const
Definition: fst.cc:98
const SymbolTable * osymbols
Definition: fst.h:62
int64_t int64
Definition: types.h:27
std::ostream & WriteType(std::ostream &strm, const T t)
Definition: util.h:155
FstReadOptions(const string &source="<unspecified>", const FstHeader *header=nullptr, const SymbolTable *isymbols=nullptr, const SymbolTable *osymbols=nullptr)
Definition: fst.cc:108
static FileReadMode ReadMode(const string &mode)
Definition: fst.cc:132
bool Read(std::istream &strm, const string &source, bool rewind=false)
Definition: fst.cc:58
string DebugString() const
Definition: fst.cc:139
string source
Definition: fst.h:57
constexpr int32 kFstMagicNumber
Definition: fst.cc:40
DEFINE_string(save_relabel_ipairs,"","Save input relabel pairs to file")
int32_t int32
Definition: types.h:26
DEFINE_int64(fst_default_cache_gc_limit, 1<< 20LL,"Cache byte size that triggers garbage collection")
FileReadMode mode
Definition: fst.h:64
std::istream & ReadType(std::istream &strm, T *t)
Definition: util.h:47
const SymbolTable * isymbols
Definition: fst.h:60
DEFINE_bool(fst_verify_properties, false,"Verify FST properties queried by TestProperties")
bool read_osymbols
Definition: fst.h:66
bool Write(std::ostream &strm, const string &source) const
Definition: fst.cc:85