FST  openfst-1.8.2.post1
OpenFst Library
farscript.h
Go to the documentation of this file.
1 // Copyright 2005-2020 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the 'License');
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an 'AS IS' BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // See www.openfst.org for extensive documentation on this weighted
16 // finite-state transducer library.
17 //
18 // Convenience file for including all of the FAR operations, or registering
19 // them for new arc types.
20 
21 #ifndef FST_EXTENSIONS_FAR_FARSCRIPT_H_
22 #define FST_EXTENSIONS_FAR_FARSCRIPT_H_
23 
24 #include <cstdint>
25 #include <string>
26 #include <vector>
27 
35 #include <fst/extensions/far/far.h>
40 #include <fst/script/arg-packs.h>
41 #include <string_view>
42 
43 namespace fst {
44 namespace script {
45 
46 // Note: it is safe to pass these strings as references because this struct is
47 // only used to pass them deeper in the call graph. Be sure you understand why
48 // this is so before using this struct for anything else!
50  const std::vector<std::string> &sources;
52  std::string_view fst_type;
53  const int32_t generate_keys;
55  const TokenType tt;
56  const std::string &symbols_source;
57  const std::string &unknown_symbol;
58  const bool keep_symbols;
59  const bool initial_symbols;
61  const std::string &key_prefix;
62  const std::string &key_suffix;
63 };
64 
65 template <class Arc>
67  FarWriter<Arc> &writer = *args->writer.GetFarWriter<Arc>();
68  ::fst::CompileStrings<Arc>(
69  args->sources, writer, args->fst_type, args->generate_keys, args->fet,
70  args->tt, args->symbols_source, args->unknown_symbol, args->keep_symbols,
72  args->key_suffix);
73 }
74 
75 void CompileStrings(const std::vector<std::string> &sources,
76  FarWriterClass &writer, std::string_view fst_type,
78  const std::string &symbols_source,
79  const std::string &unknown_symbol, bool keep_symbols,
81  const std::string &key_prefix,
82  const std::string &key_suffix);
83 
84 using FarConvertArgs =
85  std::tuple<FarReaderClass &, FarWriterClass &, std::string_view>;
86 
87 template <class Arc>
88 void Convert(FarConvertArgs *args) {
89  FarReader<Arc> &reader = *std::get<0>(*args).GetFarReader<Arc>();
90  FarWriter<Arc> &writer = *std::get<1>(*args).GetFarWriter<Arc>();
91  ::fst::Convert<Arc>(reader, writer, std::get<2>(*args));
92 }
93 
94 void Convert(FarReaderClass &reader, FarWriterClass &writer,
95  std::string_view fst_type);
96 
97 // Note: it is safe to pass these strings as references because this struct is
98 // only used to pass them deeper in the call graph. Be sure you understand why
99 // this is so before using this struct for anything else!
101  const std::vector<std::string> &sources;
103  const int32_t generate_keys;
104  const std::string &key_prefix;
105  const std::string &key_suffix;
106 };
107 
108 template <class Arc>
109 void Create(FarCreateArgs *args) {
110  FarWriter<Arc> &writer = *args->writer.GetFarWriter<Arc>();
111  ::fst::Create<Arc>(args->sources, writer, args->generate_keys,
112  args->key_prefix, args->key_suffix);
113 }
114 
115 void Create(const std::vector<std::string> &sources, FarWriterClass &writer,
116  int32_t generate_keys, const std::string &key_prefix,
117  const std::string &key_suffix);
118 
119 using FarDecodeArgs =
120  std::tuple<FarReaderClass &, FarWriterClass &, const EncodeMapperClass &>;
121 
122 template <class Arc>
123 void Decode(FarDecodeArgs *args) {
124  FarReader<Arc> &reader = *std::get<0>(*args).GetFarReader<Arc>();
125  FarWriter<Arc> &writer = *std::get<1>(*args).GetFarWriter<Arc>();
126  const EncodeMapper<Arc> &mapper = *std::get<2>(*args).GetEncodeMapper<Arc>();
127  Decode(reader, writer, mapper);
128 }
129 
130 void Decode(FarReaderClass &reader, FarWriterClass &writer,
131  const EncodeMapperClass &encoder);
132 
133 using FarEncodeArgs =
134  std::tuple<FarReaderClass &, FarWriterClass &, EncodeMapperClass *>;
135 
136 template <class Arc>
137 void Encode(FarEncodeArgs *args) {
138  FarReader<Arc> &reader = *std::get<0>(*args).GetFarReader<Arc>();
139  FarWriter<Arc> &writer = *std::get<1>(*args).GetFarWriter<Arc>();
140  EncodeMapper<Arc> *mapper = std::get<2>(*args)->GetEncodeMapper<Arc>();
141  Encode(reader, writer, mapper);
142 }
143 
144 void Encode(FarReaderClass &reader, FarWriterClass &writer,
145  EncodeMapperClass *encoder);
146 
147 using FarEqualInnerArgs = std::tuple<FarReaderClass &, FarReaderClass &, float,
148  std::string_view, std::string_view>;
149 
151 
152 template <class Arc>
153 void Equal(FarEqualArgs *args) {
154  FarReader<Arc> &reader1 = *std::get<0>(args->args).GetFarReader<Arc>();
155  FarReader<Arc> &reader2 = *std::get<1>(args->args).GetFarReader<Arc>();
156  args->retval =
157  ::fst::Equal<Arc>(reader1, reader2, std::get<2>(args->args),
158  std::get<3>(args->args), std::get<4>(args->args));
159 }
160 
161 bool Equal(FarReaderClass &reader1, FarReaderClass &reader2,
162  float delta = kDelta, std::string_view begin_key = "",
163  std::string_view end_key = "");
164 
165 using FarExtractArgs =
166  std::tuple<FarReaderClass &, int32_t, const std::string &,
167  const std::string &, const std::string &, const std::string &,
168  const std::string &>;
169 
170 template <class Arc>
171 void Extract(FarExtractArgs *args) {
172  FarReader<Arc> &reader = *std::get<0>(*args).GetFarReader<Arc>();
173  ::fst::Extract<Arc>(reader, std::get<1>(*args), std::get<2>(*args),
174  std::get<3>(*args), std::get<4>(*args),
175  std::get<5>(*args), std::get<6>(*args));
176 }
177 
178 void Extract(FarReaderClass &reader, int32_t generate_sources,
179  const std::string &keys, const std::string &key_separator,
180  const std::string &range_delimiter,
181  const std::string &source_prefix,
182  const std::string &source_suffix);
183 
184 using FarInfoArgs =
185  std::tuple<const std::vector<std::string> &, const std::string &,
186  const std::string &, const bool>;
187 
188 template <class Arc>
189 void Info(FarInfoArgs *args) {
190  ::fst::Info<Arc>(std::get<0>(*args), std::get<1>(*args),
191  std::get<2>(*args), std::get<3>(*args));
192 }
193 
194 void Info(const std::vector<std::string> &sources, const std::string &arc_type,
195  const std::string &begin_key, const std::string &end_key,
196  const bool list_fsts);
197 
198 using FarGetInfoArgs =
199  std::tuple<const std::vector<std::string> &, const std::string &,
200  const std::string &, const bool, FarInfoData *>;
201 
202 template <class Arc>
203 void GetInfo(FarGetInfoArgs *args) {
204  ::fst::GetInfo<Arc>(std::get<0>(*args), std::get<1>(*args),
205  std::get<2>(*args), std::get<3>(*args),
206  std::get<4>(*args));
207 }
208 
209 void GetInfo(const std::vector<std::string> &sources,
210  const std::string &arc_type, const std::string &begin_key,
211  const std::string &end_key, const bool list_fsts, FarInfoData *);
212 
214  std::tuple<FarReaderClass &, FarReaderClass &, float, std::string_view,
215  std::string_view>;
216 
218 
219 template <class Arc>
221  FarReader<Arc> &reader1 = *std::get<0>(args->args).GetFarReader<Arc>();
222  FarReader<Arc> &reader2 = *std::get<1>(args->args).GetFarReader<Arc>();
223  args->retval = ::fst::Isomorphic<Arc>(
224  reader1, reader2, std::get<2>(args->args), std::get<3>(args->args),
225  std::get<4>(args->args));
226 }
227 
228 bool Isomorphic(FarReaderClass &reader1, FarReaderClass &reader2,
229  float delta = kDelta, std::string_view begin_key = "",
230  std::string_view end_key = "");
231 
233  FarReaderClass &reader;
236  const std::string &begin_key;
237  const std::string &end_key;
238  const bool print_key;
239  const bool print_weight;
240  const std::string &symbols_source;
241  const bool initial_symbols;
242  const int32_t generate_sources;
243  const std::string &source_prefix;
244  const std::string &source_suffix;
245 };
246 
247 template <class Arc>
249  FarReader<Arc> &reader = *args->reader.GetFarReader<Arc>();
250  ::fst::PrintStrings<Arc>(reader, args->entry_type, args->token_type,
251  args->begin_key, args->end_key, args->print_key,
252  args->print_weight, args->symbols_source,
253  args->initial_symbols, args->generate_sources,
254  args->source_prefix, args->source_suffix);
255 }
256 
257 void PrintStrings(FarReaderClass &reader, const FarEntryType entry_type,
258  const TokenType token_type, const std::string &begin_key,
259  const std::string &end_key, const bool print_key,
260  const bool print_weight, const std::string &symbols_source,
261  const bool initial_symbols, const int32_t generate_sources,
262  const std::string &source_prefix,
263  const std::string &source_suffix);
264 
265 } // namespace script
266 } // namespace fst
267 
268 #define REGISTER_FST_FAR_OPERATIONS(ArcType) \
269  REGISTER_FST_OPERATION(CompileStrings, ArcType, FarCompileStringsArgs); \
270  REGISTER_FST_OPERATION(Create, ArcType, FarCreateArgs); \
271  REGISTER_FST_OPERATION(Equal, ArcType, FarEqualArgs); \
272  REGISTER_FST_OPERATION(Extract, ArcType, FarExtractArgs); \
273  REGISTER_FST_OPERATION(Info, ArcType, FarInfoArgs); \
274  REGISTER_FST_OPERATION(Isomorphic, ArcType, FarIsomorphicArgs); \
275  REGISTER_FST_OPERATION(PrintStrings, ArcType, FarPrintStringsArgs); \
276  REGISTER_FST_OPERATION(GetInfo, ArcType, FarGetInfoArgs)
277 
278 #endif // FST_EXTENSIONS_FAR_FARSCRIPT_H_
const FarReader< Arc > * GetFarReader() const
Definition: far-class.h:122
const std::string & symbols_source
Definition: farscript.h:56
void PrintStrings(FarReaderClass &reader, const FarEntryType entry_type, const TokenType token_type, const std::string &begin_key, const std::string &end_key, bool print_key, bool print_weight, const std::string &symbols_source, bool initial_symbols, const int32_t generate_sources, const std::string &source_prefix, const std::string &source_suffix)
Definition: farscript.cc:159
void GetInfo(const std::vector< std::string > &sources, const std::string &arc_type, const std::string &begin_key, const std::string &end_key, bool list_fsts, FarInfoData *data)
Definition: farscript.cc:129
std::tuple< const std::vector< std::string > &, const std::string &, const std::string &, const bool > FarInfoArgs
Definition: farscript.h:186
bool Equal(FarReaderClass &reader1, FarReaderClass &reader2, float delta, std::string_view begin_key, std::string_view end_key)
Definition: farscript.cc:105
std::tuple< FarReaderClass &, FarReaderClass &, float, std::string_view, std::string_view > FarIsomorphicInnerArgs
Definition: farscript.h:215
bool Isomorphic(FarReaderClass &reader1, FarReaderClass &reader2, float delta, std::string_view begin_key, std::string_view end_key)
Definition: farscript.cc:147
std::tuple< FarReaderClass &, FarWriterClass &, EncodeMapperClass * > FarEncodeArgs
Definition: farscript.h:134
const std::vector< std::string > & sources
Definition: farscript.h:50
const std::string & unknown_symbol
Definition: farscript.h:57
std::tuple< FarReaderClass &, FarWriterClass &, std::string_view > FarConvertArgs
Definition: farscript.h:85
const std::string & key_suffix
Definition: farscript.h:105
std::tuple< const std::vector< std::string > &, const std::string &, const std::string &, const bool, FarInfoData * > FarGetInfoArgs
Definition: farscript.h:200
const std::vector< std::string > & sources
Definition: farscript.h:101
std::tuple< FarReaderClass &, int32_t, const std::string &, const std::string &, const std::string &, const std::string &, const std::string & > FarExtractArgs
Definition: farscript.h:168
void Decode(FarReaderClass &reader, FarWriterClass &writer, const EncodeMapperClass &encoder)
Definition: farscript.cc:81
const std::string & source_prefix
Definition: farscript.h:243
const std::string & begin_key
Definition: farscript.h:236
void Info(const std::vector< std::string > &sources, const std::string &arc_type, const std::string &begin_key, const std::string &end_key, bool list_fsts)
Definition: farscript.cc:138
FarWriterClass & writer
Definition: farscript.h:102
std::tuple< FarReaderClass &, FarWriterClass &, const EncodeMapperClass & > FarDecodeArgs
Definition: farscript.h:120
void CompileStrings(const std::vector< std::string > &sources, FarWriterClass &writer, std::string_view fst_type, int32_t generate_keys, FarEntryType fet, TokenType tt, const std::string &symbols_source, const std::string &unknown_symbol, bool keep_symbols, bool initial_symbols, bool allow_negative_labels, const std::string &key_prefix, const std::string &key_suffix)
Definition: farscript.cc:37
void Encode(FarReaderClass &reader, FarWriterClass &writer, EncodeMapperClass *encoder)
Definition: farscript.cc:93
const std::string & key_suffix
Definition: farscript.h:62
const int32_t generate_keys
Definition: farscript.h:103
TokenType
Definition: string.h:47
std::tuple< FarReaderClass &, FarReaderClass &, float, std::string_view, std::string_view > FarEqualInnerArgs
Definition: farscript.h:148
const FarWriter< Arc > * GetFarWriter() const
Definition: far-class.h:243
void Create(const std::vector< std::string > &sources, FarWriterClass &writer, const int32_t generate_keys, const std::string &key_prefix, const std::string &key_suffix)
Definition: farscript.cc:72
const std::string & symbols_source
Definition: farscript.h:240
const std::string & end_key
Definition: farscript.h:237
const std::string & source_suffix
Definition: farscript.h:244
const std::string & key_prefix
Definition: farscript.h:104
const FarEntryType entry_type
Definition: farscript.h:234
void Extract(FarReaderClass &reader, int32_t generate_sources, const std::string &keys, const std::string &key_separator, const std::string &range_delimiter, const std::string &source_prefix, const std::string &source_suffix)
Definition: farscript.cc:116
FarEntryType
Definition: far.h:41
constexpr float kDelta
Definition: weight.h:130
void Convert(FarReaderClass &reader, FarWriterClass &writer, std::string_view fst_type)
Definition: farscript.cc:64
const std::string & key_prefix
Definition: farscript.h:61