10 #include <sys/types.h> 92 bool write_header =
true,
bool write_isymbols =
true,
93 bool write_osymbols =
true,
94 bool align = FLAGS_fst_align,
95 bool stream_write =
false)
97 write_header(write_header),
98 write_isymbols(write_isymbols),
99 write_osymbols(write_osymbols),
101 stream_write(stream_write) {}
116 FstHeader() : version_(0), flags_(0), properties_(0), start_(-1),
117 numstates_(0), numarcs_(0) {}
119 const string &
FstType()
const {
return fsttype_; }
121 const string &
ArcType()
const {
return arctype_; }
151 bool Read(std::istream &strm,
const string &
source,
152 bool rewind =
false);
154 bool Write(std::ostream &strm,
const string &source)
const;
195 virtual StateId Start()
const = 0;
212 virtual uint64 Properties(
uint64 mask,
bool test)
const = 0;
215 virtual const string &Type()
const = 0;
231 virtual Fst<Arc> *Copy(
bool safe =
false)
const = 0;
240 if (!hdr.
Read(strm, opts.
source))
return nullptr;
243 const auto &fst_type = hdr.
FstType();
246 LOG(ERROR) <<
"Fst::Read: Unknown FST type " << fst_type
247 <<
" (arc type = " << Arc::Type() <<
"): " << ropts.
source;
250 return reader(strm, ropts);
256 if (!filename.empty()) {
257 std::ifstream strm(filename,
258 std::ios_base::in | std::ios_base::binary);
260 LOG(ERROR) <<
"Fst::Read: Can't open file: " << filename;
271 LOG(ERROR) <<
"Fst::Write: No write stream method for " << Type()
278 virtual bool Write(
const string &filename)
const {
279 LOG(ERROR) <<
"Fst::Write: No write filename method for " << Type()
285 virtual const SymbolTable *InputSymbols()
const = 0;
288 virtual const SymbolTable *OutputSymbols()
const = 0;
304 if (!filename.empty()) {
305 std::ofstream strm(filename,
306 std::ios_base::out | std::ios_base::binary);
308 LOG(ERROR) <<
"Fst::Write: Can't open file: " << filename;
312 if (!val)
LOG(ERROR) <<
"Fst::Write failed: " << filename;
335 virtual bool Done()
const = 0;
337 virtual StateId Value()
const = 0;
339 virtual void Next() = 0;
341 virtual void Reset() = 0;
374 using Arc =
typename FST::Arc;
378 fst.InitStateIterator(&data_);
384 return data_.base ? data_.base->Done() : s_ >= data_.nstates;
387 StateId Value()
const {
return data_.base ? data_.base->Value() : s_; }
411 static constexpr
uint32 kArcILabelValue =
413 static constexpr
uint32 kArcOLabelValue = 0x0002;
414 static constexpr
uint32 kArcWeightValue = 0x0004;
415 static constexpr
uint32 kArcNextStateValue =
417 static constexpr
uint32 kArcNoCache = 0x0010;
419 static constexpr
uint32 kArcValueFlags =
420 kArcILabelValue | kArcOLabelValue | kArcWeightValue | kArcNextStateValue;
422 static constexpr
uint32 kArcFlags = kArcValueFlags | kArcNoCache;
434 virtual bool Done()
const = 0;
436 virtual const Arc &Value()
const = 0;
438 virtual void Next() = 0;
440 virtual size_t Position()
const = 0;
442 virtual void Reset() = 0;
444 virtual void Seek(
size_t) = 0;
446 virtual uint32 Flags()
const = 0;
455 : base(nullptr), arcs(nullptr), narcs(0), ref_count(nullptr) {}
480 using Arc =
typename FST::Arc;
484 fst.InitArcIterator(s, &data_);
488 if (data_.ref_count) ++(*data_.ref_count);
494 }
else if (data_.ref_count) {
495 --(*data_.ref_count);
500 return data_.base ? data_.base->Done() : i_ >= data_.narcs;
504 return data_.base ? data_.base->Value() : data_.arcs[i_];
531 size_t Position()
const {
return data_.base ? data_.base->Position() : i_; }
535 return data_.base->Flags();
537 return kArcValueFlags;
542 if (data_.base) data_.base->SetFlags(flags, mask);
556 void *
operator new(
size_t size,
558 return pool->Allocate();
585 inline typename F::Arc::Weight
Final(
const F &
fst,
typename F::Arc::StateId s) {
586 return fst.F::Final(s);
590 inline ssize_t
NumArcs(
const F &
fst,
typename F::Arc::StateId s) {
591 return fst.F::NumArcs(s);
596 return fst.F::NumInputEpsilons(s);
601 return fst.F::NumOutputEpsilons(s);
608 typename Arc::StateId s) {
643 : properties_(impl.properties_),
645 isymbols_(impl.isymbols_ ? impl.isymbols_->Copy() : nullptr),
646 osymbols_(impl.osymbols_ ? impl.osymbols_->Copy() : nullptr) {}
655 isymbols_ = impl.isymbols_ ? impl.isymbols_->Copy() :
nullptr;
656 osymbols_ = impl.osymbols_ ? impl.osymbols_->Copy() :
nullptr;
662 const string &
Type()
const {
return type_; }
664 void SetType(
const string &type) { type_ = type; }
672 properties_ |= props;
676 properties_ &= ~mask |
kError;
677 properties_ |= props & mask;
683 FSTERROR() <<
"FstImpl::SetProperties() const: Can only set kError";
697 isymbols_.reset(isyms ? isyms->
Copy() :
nullptr);
701 osymbols_.reset(osyms ? osyms->
Copy() :
nullptr);
721 int32 file_flags = 0;
742 const string &type,
uint64 properties,
749 int32 file_flags = 0;
775 const string &type,
uint64 properties,
777 strm.seekp(header_offset);
779 LOG(ERROR) <<
"Fst::UpdateFstHeader: Write failed: " << opts.
source;
782 WriteFstHeader(fst, strm, opts, version, type, properties, hdr);
784 LOG(ERROR) <<
"Fst::UpdateFstHeader: Write failed: " << opts.
source;
787 strm.seekp(0, std::ios_base::end);
789 LOG(ERROR) <<
"Fst::UpdateFstHeader: Write failed: " << opts.
source;
800 std::unique_ptr<SymbolTable> isymbols_;
801 std::unique_ptr<SymbolTable> osymbols_;
809 FstImpl<Arc> &&) noexcept = default;
816 }
else if (!hdr->Read(strm, opts.source)) {
820 LOG(INFO) <<
"FstImpl::ReadHeader: source: " << opts.source
821 <<
", fst_type: " << hdr->FstType()
822 <<
", arc_type: " << Arc::Type()
823 <<
", version: " << hdr->Version()
824 <<
", flags: " << hdr->GetFlags();
826 if (hdr->FstType() != type_) {
827 LOG(ERROR) <<
"FstImpl::ReadHeader: FST not of type " << type_
828 <<
": " << opts.source;
831 if (hdr->ArcType() != Arc::Type()) {
832 LOG(ERROR) <<
"FstImpl::ReadHeader: Arc not of type " << Arc::Type()
833 <<
": " << opts.source;
836 if (hdr->Version() < min_version) {
837 LOG(ERROR) <<
"FstImpl::ReadHeader: Obsolete " << type_
838 <<
" FST version: " << opts.source;
841 properties_ = hdr->Properties();
846 if (!opts.read_isymbols) SetInputSymbols(
nullptr);
851 if (!opts.read_osymbols) SetOutputSymbols(
nullptr);
853 isymbols_.reset(opts.isymbols->Copy());
856 osymbols_.reset(opts.osymbols->Copy());
868 template <
class Impl,
class FST = Fst<
typename Impl::Arc>>
871 using Arc =
typename Impl::Arc;
882 return impl_->NumInputEpsilons(s);
886 return impl_->NumOutputEpsilons(s);
892 impl_->SetProperties(testprops, knownprops);
893 return testprops & mask;
895 return impl_->Properties(mask);
899 const string &
Type()
const override {
return impl_->Type(); }
902 return impl_->InputSymbols();
906 return impl_->OutputSymbols();
910 explicit ImplToFst(std::shared_ptr<Impl> impl) : impl_(
std::move(impl)) {}
916 impl_ = std::make_shared<Impl>(*(fst.impl_));
927 : impl_(
std::move(fst.impl_)) {
928 fst.impl_ = std::make_shared<Impl>();
938 impl_ = std::move(fst.impl_);
939 fst.impl_ = std::make_shared<Impl>();
945 const Impl *
GetImpl()
const {
return impl_.get(); }
952 bool Unique()
const {
return impl_.unique(); }
954 void SetImpl(std::shared_ptr<Impl> impl) { impl_ = std::move(impl); }
957 template <
class IFST,
class OFST>
958 friend void Cast(
const IFST &ifst, OFST *ofst);
960 std::shared_ptr<Impl> impl_;
970 template <
class IFST,
class OFST>
971 void Cast(
const IFST &ifst, OFST *ofst) {
972 using OImpl =
typename OFST::Impl;
973 ofst->impl_ = std::shared_ptr<OImpl>(ifst.impl_,
974 reinterpret_cast<OImpl *
>(ifst.impl_.get()));
983 std::ostringstream ostrm;
984 fst.
Write(ostrm, options);
1001 std::istringstream istrm(s);
1007 #endif // FST_FST_H_
ImplToFst< Impl, FST > & operator=(ImplToFst< Impl, FST > &&fst) noexcept
void SetProperties(uint64 props, uint64 mask) const
const SymbolTable * OutputSymbols() const override
typename ArcMapFst< Arc, Arc, EncodeMapper< Arc > >::Arc Arc
void Cast(const F &, G *)
std::shared_ptr< Impl > GetSharedImpl() const
virtual bool Write(std::ostream &strm) const
virtual size_t NumArcs(StateId) const =0
Fst< Arc > * StringToFst(const string &s)
FstImpl & operator=(const FstImpl< Arc > &impl)
bool IsFstHeader(std::istream &, const string &)
uint64 TestProperties(const Fst< Arc > &fst, uint64 mask, uint64 *known)
const SymbolTable * OutputSymbols() const
void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags, const char *src="")
typename Arc::Weight Weight
virtual SymbolTable * Copy() const
virtual bool Write(std::ostream &strm, const FstWriteOptions &opts) const
size_t NumArcs(const Fst< Arc > &fst, typename Arc::StateId s)
StateId Start() const override
virtual Weight Final(StateId) const =0
void SetOutputSymbols(const SymbolTable *osyms)
typename RandGenFst< FromArc, ToArc, Sampler >::Arc::StateId StateId
typename ArcMapFst< Arc, Arc, EncodeMapper< Arc > >::Arc Arc
typename FST::Arc::StateId StateId
virtual uint64 Properties(uint64 mask) const
static bool UpdateFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, const string &type, uint64 properties, FstHeader *hdr, size_t header_offset)
size_t NumArcs(StateId s) const override
const SymbolTable * osymbols
virtual uint64 Properties() const
SymbolTable * InputSymbols()
const Arc & Value() const
static Fst< Arc > * Read(std::istream &strm, const FstReadOptions &opts)
static FstRegister< Arc > * GetRegister()
virtual size_t NumInputEpsilons(StateId) const =0
size_t NumInputEpsilons(StateId s) const override
void SetType(const string &type)
ArcIterator(const FST &fst, StateId s)
const string & Type() const override
FstReadOptions(const string &source="<unspecified>", const FstHeader *header=nullptr, const SymbolTable *isymbols=nullptr, const SymbolTable *osymbols=nullptr)
size_t NumOutputEpsilons(const Fst< Arc > &fst, typename Arc::StateId s)
SymbolTable * OutputSymbols()
void SetProperties(uint64 props, uint64 mask)
bool WriteFile(const string &filename) const
StateIteratorBase< Arc > * base
static SymbolTable * Read(std::istream &strm, const SymbolTableReadOptions &opts)
ImplToFst(const ImplToFst< Impl, FST > &fst, bool safe)
virtual ~StateIteratorBase()
typename Arc::StateId StateId
static FileReadMode ReadMode(const string &mode)
ImplToFst(ImplToFst< Impl, FST > &&fst) noexcept
typename Arc::StateId StateId
void SetFlags(uint32 flags, uint32 mask)
size_t NumOutputEpsilons(StateId s) const override
static void WriteFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, const string &type, uint64 properties, FstHeader *hdr)
string DebugString() const
void WriteHeader(std::ostream &strm, const FstWriteOptions &opts, int version, FstHeader *hdr) const
const string & Type() const
typename S::Arc::Weight Weight
FstWriteOptions(const string &source="<unspecifed>", bool write_header=true, bool write_isymbols=true, bool write_osymbols=true, bool align=FLAGS_fst_align, bool stream_write=false)
static Fst< Arc > * Read(const string &filename)
const SymbolTable * InputSymbols() const override
void SetInputSymbols(const SymbolTable *isyms)
void SetImpl(std::shared_ptr< Impl > impl)
virtual bool Write(const string &filename) const
virtual ~ArcIteratorBase()
ImplToFst(const ImplToFst< Impl, FST > &fst)
FstImpl(const FstImpl< Arc > &impl)
Arc::Weight Final(const Fst< Arc > &fst, typename Arc::StateId s)
typename internal::SynchronizeFstImpl< A >::Arc Arc
virtual const SymbolTable * InputSymbols() const =0
const SymbolTable * InputSymbols() const
Weight Final(StateId s) const override
void SetProperties(uint64 props)
ImplToFst< Impl, FST > & operator=(const ImplToFst< Impl, FST > &fst)
ImplToFst(std::shared_ptr< Impl > impl)
virtual MatcherBase< Arc > * InitMatcher(MatchType match_type) const
typename Arc::StateId StateId
ArcIteratorBase< Arc > * base
typename Arc::StateId StateId
const SymbolTable * isymbols
string FstToString(const Fst< Arc > &fst, const FstWriteOptions &options=FstWriteOptions("FstToString"))
Impl * GetMutableImpl() const
size_t NumInputEpsilons(const Fst< Arc > &fst, typename Arc::StateId s)
uint64 Properties(uint64 mask, bool test) const override
virtual size_t NumOutputEpsilons(StateId) const =0
void Destroy(ArcIterator< FST > *aiter, MemoryPool< ArcIterator< FST >> *pool)
typename S::Arc::StateId StateId
const Impl * GetImpl() const
ArcIterator(const ArcIteratorData< Arc > &data)
virtual const SymbolTable * OutputSymbols() const =0
StateIterator(const FST &fst)