20 #ifndef FST_SCRIPT_COMPILE_IMPL_H_ 21 #define FST_SCRIPT_COMPILE_IMPL_H_ 38 #include <unordered_map> 39 #include <string_view> 51 using Label =
typename Arc::Label;
61 const SymbolTable *ssyms,
bool accep,
bool ikeep,
bool okeep,
63 std::unique_ptr<SymbolTable> misyms(isyms ? isyms->
Copy() :
nullptr);
64 std::unique_ptr<SymbolTable> mosyms(osyms ? osyms->
Copy() :
nullptr);
65 std::unique_ptr<SymbolTable> mssyms(ssyms ? ssyms->
Copy() :
nullptr);
66 Init(istrm, source, misyms.get(), mosyms.get(), mssyms.get(), accep, ikeep,
72 bool okeep,
bool nkeep,
bool add_symbols) {
73 Init(istrm, source, isyms, osyms, ssyms, accep, ikeep, okeep, nkeep,
79 bool okeep,
bool nkeep,
bool add_symbols) {
81 source_ = std::string(source);
86 keep_state_numbering_ = nkeep;
87 add_symbols_ = add_symbols;
88 bool start_state_populated =
false;
90 const std::string separator =
91 FST_FLAGS_fst_field_separator +
"\n";
92 while (istrm.getline(line, kLineLen)) {
94 const std::vector<std::string_view> col =
96 if (col.empty() || col[0].empty())
continue;
97 if (col.size() > 5 || (col.size() > 4 && accep) ||
98 (col.size() == 3 && !accep)) {
99 FSTERROR() <<
"FstCompiler: Bad number of columns, source = " << source_
100 <<
", line = " << nline_;
104 StateId s = StrToStateId(col[0]);
106 if (!start_state_populated) {
108 start_state_populated =
true;
112 switch (col.size()) {
117 fst_.
SetFinal(s, StrToWeight(col[1],
true));
120 arc.nextstate = d = StrToStateId(col[1]);
121 arc.ilabel = StrToILabel(col[2]);
122 arc.olabel = arc.ilabel;
123 arc.weight = Weight::One();
127 arc.nextstate = d = StrToStateId(col[1]);
128 arc.ilabel = StrToILabel(col[2]);
130 arc.olabel = arc.ilabel;
131 arc.weight = StrToWeight(col[3],
true);
133 arc.olabel = StrToOLabel(col[3]);
134 arc.weight = Weight::One();
139 arc.nextstate = d = StrToStateId(col[1]);
140 arc.ilabel = StrToILabel(col[2]);
141 arc.olabel = StrToOLabel(col[3]);
142 arc.weight = StrToWeight(col[4],
true);
155 static constexpr
int kLineLen = 8096;
158 std::string_view name)
const {
168 FSTERROR() <<
"FstCompiler: Symbol \"" << s
169 <<
"\" is not mapped to any integer " << name
170 <<
", symbol table = " << syms->
Name()
171 <<
", source = " << source_ <<
", line = " << nline_;
176 if (!maybe_n.has_value()) {
177 FSTERROR() <<
"FstCompiler: Bad " << name <<
" integer = \"" << s
178 <<
"\", source = " << source_ <<
", line = " << nline_;
186 StateId StrToStateId(std::string_view s) {
187 StateId n = StrToId(s, ssyms_,
"state ID");
188 if (keep_state_numbering_)
return n;
190 const auto it = states_.find(n);
191 if (it == states_.end()) {
192 states_[n] = nstates_;
199 StateId StrToILabel(std::string_view s)
const {
200 return StrToId(s, isyms_,
"arc ilabel");
203 StateId StrToOLabel(std::string_view s)
const {
204 return StrToId(s, osyms_,
"arc olabel");
207 Weight StrToWeight(std::string_view s,
bool allow_zero)
const {
209 std::istringstream strm(std::string{s});
211 if (!strm || (!allow_zero && w == Weight::Zero())) {
212 FSTERROR() <<
"FstCompiler: Bad weight = \"" << s
213 <<
"\", source = " << source_ <<
", line = " << nline_;
215 w = Weight::NoWeight();
226 std::unordered_map<StateId, StateId> states_;
228 bool keep_state_numbering_;
237 #endif // FST_SCRIPT_COMPILE_IMPL_H_ const std::string & Name() const
int64_t AddSymbol(std::string_view symbol, int64_t key)
void AddArc(StateId s, const Arc &arc) override
void SetStart(StateId s) override
FstCompiler(std::istream &istrm, std::string_view source, const SymbolTable *isyms, const SymbolTable *osyms, const SymbolTable *ssyms, bool accep, bool ikeep, bool okeep, bool nkeep)
typename Arc::StateId StateId
void SetFinal(StateId s, Weight weight=Weight::One()) override
void Init(std::istream &istrm, std::string_view source, SymbolTable *isyms, SymbolTable *osyms, SymbolTable *ssyms, bool accep, bool ikeep, bool okeep, bool nkeep, bool add_symbols)
virtual SymbolTable * Copy() const
constexpr uint64_t kError
void SetProperties(uint64_t props, uint64_t mask) override
typename Arc::Weight Weight
typename Arc::Label Label
FstCompiler(std::istream &istrm, std::string_view source, SymbolTable *isyms, SymbolTable *osyms, SymbolTable *ssyms, bool accep, bool ikeep, bool okeep, bool nkeep, bool add_symbols)
constexpr int64_t kNoSymbol
internal::StringSplitter StrSplit(std::string_view full, ByAnyChar delim)
StateId AddState() override
std::optional< int64_t > ParseInt64(std::string_view s, int base=10)
void SetOutputSymbols(const SymbolTable *osyms) override
DECLARE_string(fst_field_separator)
void SetInputSymbols(const SymbolTable *isyms) override
StateId NumStates() const override
std::string Find(int64_t key) const
const VectorFst< Arc > & Fst() const