20 #ifndef FST_EXTENSIONS_FAR_FAR_H_ 21 #define FST_EXTENSIONS_FAR_FAR_H_ 45 #include <string_view> 61 inline bool IsFst(std::string_view source) {
62 std::ifstream strm(std::string(source),
63 std::ios_base::in | std::ios_base::binary);
64 if (!strm)
return false;
65 int32_t magic_number = 0;
74 const std::string &
ArcType()
const {
return arctype_; }
78 bool Read(
const std::string &source) {
84 arctype_ =
"standard";
96 }
else if (
IsFst(source)) {
98 std::ifstream istrm(source,
99 std::ios_base::in | std::ios_base::binary);
100 if (!fsthdr.
Read(istrm, source))
return false;
109 std::string arctype_;
124 virtual void Add(std::string_view key,
const Fst<Arc> &
fst) = 0;
126 virtual FarType Type()
const = 0;
128 virtual bool Error()
const = 0;
144 static FarReader *Open(
const std::string &source);
148 static FarReader *Open(
const std::vector<std::string> &sources);
151 virtual void Reset() = 0;
154 virtual bool Find(std::string_view key) = 0;
157 virtual bool Done()
const = 0;
160 virtual void Next() = 0;
164 virtual const std::string &GetKey()
const = 0;
168 virtual const Fst<Arc> *GetFst()
const = 0;
170 virtual FarType Type()
const = 0;
172 virtual bool Error()
const = 0;
199 writer_->Add(key,
fst);
204 bool Error() const final {
return writer_->Error(); }
224 writer_->Add(key,
fst);
229 bool Error() const final {
return writer_->Error(); }
244 : source_(source), error_(false), written_(false) {}
252 LOG(WARNING) <<
"FstFarWriter::Add: only one FST supported," 253 <<
" subsequent entries discarded.";
255 error_ = !
fst.Write(source_);
262 bool Error() const final {
return error_; }
285 LOG(ERROR) <<
"FarWriter::Create: Unknown FAR type";
307 if (!reader || reader->Error())
return nullptr;
314 if (!reader || reader->Error())
return nullptr;
318 void Reset() final { reader_->Reset(); }
320 bool Find(std::string_view key)
final {
return reader_->Find(key); }
322 bool Done() const final {
return reader_->Done(); }
324 void Next() final {
return reader_->Next(); }
326 const std::string &
GetKey() const final {
return reader_->GetKey(); }
332 bool Error() const final {
return reader_->Error(); }
337 : reader_(std::move(reader)) {}
350 if (!reader || reader->Error())
return nullptr;
357 if (!reader || reader->Error())
return nullptr;
361 void Reset() final { reader_->Reset(); }
363 bool Find(std::string_view key)
final {
return reader_->Find(key); }
365 bool Done() const final {
return reader_->Done(); }
367 void Next() final {
return reader_->Next(); }
369 const std::string &
GetKey() const final {
return reader_->GetKey(); }
375 bool Error() const final {
return reader_->Error(); }
380 : reader_(std::move(reader)) {}
391 std::vector<std::string> sources;
392 sources.push_back(std::string(source));
401 : keys_(sources), has_stdin_(false), pos_(0), error_(false) {
402 std::sort(keys_.begin(), keys_.end());
403 streams_.resize(keys_.size(),
nullptr);
404 for (
size_t i = 0; i < keys_.size(); ++i) {
405 if (keys_[i].empty()) {
407 streams_[i] = &std::cin;
410 FSTERROR() <<
"FstFarReader::FstFarReader: standard input should " 411 "only appear once in the input file list";
416 streams_[i] =
new std::ifstream(
417 keys_[i], std::ios_base::in | std::ios_base::binary);
418 if (streams_[i]->fail()) {
419 FSTERROR() <<
"FstFarReader::FstFarReader: Error reading file: " 426 if (pos_ >= keys_.size())
return;
433 <<
"FstFarReader::Reset: Operation not supported on standard input";
441 bool Find(std::string_view key)
final {
444 <<
"FstFarReader::Find: Operation not supported on standard input";
453 bool Done() const final {
return error_ || pos_ >= keys_.size(); }
460 const std::string &
GetKey() const final {
return keys_[pos_]; }
466 bool Error() const final {
return error_; }
469 for (
size_t i = 0; i < keys_.size(); ++i) {
470 if (streams_[i] != &std::cin) {
479 if (pos_ >= keys_.size())
return;
480 streams_[pos_]->seekg(0);
483 FSTERROR() <<
"FstFarReader: Error reading Fst from: " << keys_[pos_];
488 std::vector<std::string> keys_;
489 std::vector<std::istream *> streams_;
492 mutable std::unique_ptr<Fst<Arc>> fst_;
504 else if (
IsFst(source))
511 if (!sources.empty() && sources[0].empty())
513 else if (!sources.empty() &&
IsSTTable(sources[0]))
515 else if (!sources.empty() &&
IsSTList(sources[0]))
517 else if (!sources.empty() &&
IsFst(sources[0]))
524 #endif // FST_EXTENSIONS_FAR_FAR_H_
constexpr int32_t kFstMagicNumber
static STListFarReader * Open(const std::vector< std::string > &sources)
bool Find(std::string_view key) final
FarType Type() const final
const Fst< Arc > * GetFst() const final
void Create(const std::vector< std::string > &sources, FarWriter< Arc > &writer, int32_t generate_keys, const std::string &key_prefix, const std::string &key_suffix)
static Fst * Read(std::istream &strm, const FstReadOptions &opts)
static FarReader * Open(const std::string &source)
const std::string & GetKey() const final
static STTableFarWriter * Create(std::string_view source)
virtual bool Write(std::ostream &strm, const FstWriteOptions &opts) const
static const std::string & Type()
static STTableFarReader * Open(const std::vector< std::string > &sources)
std::unique_ptr< T > WrapUnique(T *ptr)
FarType Type() const final
bool ReadSTListHeader(const std::string &source, Header *header)
bool Find(std::string_view key) final
const Fst< Arc > * GetFst() const final
void Add(std::string_view key, const Fst< Arc > &fst) final
Fst< Arc > * operator()(std::istream &strm, const FstReadOptions &options=FstReadOptions()) const
FarType Type() const final
bool Find(std::string_view key) final
static STListFarWriter * Create(std::string_view source)
bool ReadSTTableHeader(const std::string &source, Header *header)
static STTableFarReader * Open(std::string_view source)
FarType Type() const final
static FstFarReader * Open(std::string_view source)
bool IsSTList(std::string_view source)
FarType Type() const final
static FstFarReader * Open(const std::vector< std::string > &sources)
const Fst< Arc > * GetFst() const final
std::istream & ReadType(std::istream &strm, T *t)
static STListFarReader * Open(std::string_view source)
static FarWriter * Create(std::string_view source, FarType type=FarType::DEFAULT)
bool IsSTTable(std::string_view source)
FstFarReader(const std::vector< std::string > &sources)
void Add(std::string_view key, const Fst< A > &fst) final
const std::string & GetKey() const final
void operator()(std::ostream &strm, const Fst< Arc > &fst) const
const std::string & GetKey() const final
void Add(std::string_view key, const Fst< Arc > &fst) final
bool IsFst(std::string_view source)
FstFarWriter(std::string_view source)
static FstFarWriter * Create(std::string_view source)
FarType Type() const final