7 #ifndef FST_CONST_FST_H_ 8 #define FST_CONST_FST_H_ 27 template <
class A,
class Un
signed>
30 template <
class F,
class G>
31 void Cast(
const F &, G *);
37 template <
class A,
class Un
signed>
56 string type =
"const";
57 if (
sizeof(Unsigned) !=
sizeof(
uint32)) {
58 type += std::to_string(CHAR_BIT *
sizeof(Unsigned));
92 data->
arcs = arcs_ + states_[s].pos;
93 data->
narcs = states_[s].narcs;
109 ConstState() : weight(
Weight::Zero()) {}
116 static constexpr
int kFileVersion = 2;
118 static constexpr
int kAlignedFileVersion = 1;
120 static constexpr
int kMinFileVersion = 1;
122 std::unique_ptr<MappedFile> states_region_;
123 std::unique_ptr<MappedFile> arcs_region_;
134 template <
class Arc,
class Un
signed>
137 template <
class Arc,
class Un
signed>
140 template <
class Arc,
class Un
signed>
143 template <
class Arc,
class Un
signed>
146 template <
class Arc,
class Un
signed>
148 : narcs_(0), nstates_(0) {
149 string type =
"const";
150 if (
sizeof(Unsigned) !=
sizeof(
uint32)) {
151 type += std::to_string(CHAR_BIT *
sizeof(Unsigned));
156 start_ = fst.
Start();
160 narcs_ += fst.
NumArcs(siter.Value());
164 states_ =
reinterpret_cast<ConstState *
>(states_region_->mutable_data());
165 arcs_ =
reinterpret_cast<Arc *
>(arcs_region_->mutable_data());
167 for (
StateId s = 0; s < nstates_; ++s) {
168 states_[s].weight = fst.
Final(s);
169 states_[s].pos = pos;
170 states_[s].narcs = 0;
171 states_[s].niepsilons = 0;
172 states_[s].noepsilons = 0;
174 const auto &arc = aiter.Value();
176 if (arc.ilabel == 0) ++states_[s].niepsilons;
177 if (arc.olabel == 0) ++states_[s].noepsilons;
191 template <
class Arc,
class Un
signed>
195 std::unique_ptr<ConstFstImpl<Arc, Unsigned>> impl(
198 if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr))
return nullptr;
199 impl->start_ = hdr.
Start();
203 if (hdr.
Version() == kAlignedFileVersion) {
207 LOG(ERROR) <<
"ConstFst::Read: Alignment failed: " << opts.
source;
210 size_t b = impl->nstates_ *
sizeof(ConstState);
211 impl->states_region_.reset(
213 if (!strm || !impl->states_region_) {
214 LOG(ERROR) <<
"ConstFst::Read: Read failed: " << opts.
source;
218 reinterpret_cast<ConstState *
>(impl->states_region_->mutable_data());
220 LOG(ERROR) <<
"ConstFst::Read: Alignment failed: " << opts.
source;
223 b = impl->narcs_ *
sizeof(
Arc);
224 impl->arcs_region_.reset(
226 if (!strm || !impl->arcs_region_) {
227 LOG(ERROR) <<
"ConstFst::Read: Read failed: " << opts.
source;
230 impl->arcs_ =
reinterpret_cast<Arc *
>(impl->arcs_region_->mutable_data());
231 return impl.release();
240 template <
class A,
class Un
signed>
252 template <
class F,
class G>
253 void friend Cast(
const F &, G *);
271 auto *impl = Impl::Read(strm, opts);
285 return WriteFst(*
this, strm, opts);
288 bool Write(
const string &filename)
const override {
293 static bool WriteFst(
const FST &
fst, std::ostream &strm,
297 GetImpl()->InitStateIterator(data);
301 GetImpl()->InitArcIterator(s, data);
305 explicit ConstFst(std::shared_ptr<Impl> impl)
311 static const Impl *GetImplIfConstFst(
const ConstFst &const_fst) {
316 template <
typename FST>
317 static Impl *GetImplIfConstFst(
const FST &fst) {
326 template <
class Arc,
class Un
signed>
330 const auto file_version =
334 size_t num_states = 0;
335 size_t start_offset = 0;
336 bool update_header =
true;
337 if (
const auto *impl = GetImplIfConstFst(fst)) {
338 num_arcs = impl->narcs_;
339 num_states = impl->nstates_;
340 update_header =
false;
341 }
else if (opts.
stream_write || (start_offset = strm.tellp()) == -1) {
346 num_arcs += fst.NumArcs(siter.Value());
349 update_header =
false;
355 string type =
"const";
356 if (
sizeof(Unsigned) !=
sizeof(
uint32)) {
357 type += std::to_string(CHAR_BIT *
sizeof(Unsigned));
359 const auto properties =
365 LOG(ERROR) <<
"Could not align file during write after header";
372 const auto s = siter.Value();
373 state.weight = fst.
Final(s);
378 strm.write(reinterpret_cast<const char *>(&state),
sizeof(state));
385 LOG(ERROR) <<
"Could not align file during write after writing states";
390 const auto &arc = aiter.Value();
392 #ifdef MEMORY_SANITIZER 395 ANNOTATE_MEMORY_IS_INITIALIZED(reinterpret_cast<const char *>(&arc),
399 strm.write(reinterpret_cast<const char *>(&arc),
sizeof(arc));
404 LOG(ERROR) <<
"ConstFst::WriteFst: write failed: " << opts.
source;
409 fst, strm, opts, file_version, type, properties, &hdr, start_offset);
412 LOG(ERROR) <<
"Inconsistent number of states observed during write";
415 if (hdr.
NumArcs() != num_arcs) {
416 LOG(ERROR) <<
"Inconsistent number of arcs observed during write";
425 template <
class Arc,
class Un
signed>
431 : nstates_(fst.GetImpl()->
NumStates()), s_(0) {}
433 bool Done()
const {
return s_ >= nstates_; }
448 template <
class Arc,
class Un
signed>
454 : arcs_(fst.GetImpl()->
Arcs(s)),
455 narcs_(fst.GetImpl()->
NumArcs(s)),
458 bool Done()
const {
return i_ >= narcs_; }
468 void Seek(
size_t a) { i_ = a; }
485 #endif // FST_CONST_FST_H_
static Impl * Read(std::istream &strm, const FstReadOptions &opts)
typename Arc::Weight Weight
void Cast(const F &, G *)
virtual size_t NumArcs(StateId) const =0
constexpr uint64 kUnweightedCycles
void SetFlags(uint32, uint32)
virtual Weight Final(StateId) const =0
void SetOutputSymbols(const SymbolTable *osyms)
constexpr uint64 kWeightedCycles
static ConstFst< A, Unsigned > * Read(std::istream &strm, const FstReadOptions &opts)
static bool UpdateFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, const string &type, uint64 properties, FstHeader *hdr, size_t header_offset)
typename Arc::StateId StateId
size_t NumArcs(StateId s) const override
static bool WriteFst(const FST &fst, std::ostream &strm, const FstWriteOptions &opts)
constexpr uint64 kCopyProperties
constexpr uint64 kExpanded
virtual uint64 Properties(uint64 mask, bool test) const =0
size_t NumInputEpsilons(StateId s) const override
void SetType(const string &type)
constexpr uint32 Flags() const
ConstFst(const Fst< Arc > &fst)
ArcIterator(const ConstFst< Arc, Unsigned > &fst, StateId s)
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data) const override
bool WriteFile(const string &filename) const
StateIteratorBase< Arc > * base
const Arc * Arcs(StateId s) const
bool AlignOutput(std::ostream &strm)
uint64 CheckProperties(const Fst< Arc > &fst, uint64 check_mask, uint64 test_mask)
typename Arc::StateId StateId
typename Impl::ConstState ConstState
size_t NumArcs(StateId s) const
ConstFst< A, Unsigned > * Copy(bool safe=false) const override
size_t NumOutputEpsilons(StateId s) const override
static void WriteFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, const string &type, uint64 properties, FstHeader *hdr)
size_t NumInputEpsilons(StateId s) const
virtual StateId Start() const =0
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data) const
static MappedFile * Map(std::istream *istrm, bool memorymap, const string &source, size_t size)
StateIterator(const ConstFst< Arc, Unsigned > &fst)
void SetInputSymbols(const SymbolTable *isyms)
typename Arc::StateId StateId
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override
static ConstFst< A, Unsigned > * Read(const string &filename)
virtual const SymbolTable * InputSymbols() const =0
typename Arc::StateId StateId
Weight Final(StateId s) const override
void SetProperties(uint64 props)
static MappedFile * Allocate(size_t size, int align=kArchAlignment)
size_t NumOutputEpsilons(StateId s) const
bool Write(const string &filename) const override
ConstFst(const ConstFst< A, Unsigned > &fst, bool safe=false)
constexpr uint64 kNullProperties
ArcIteratorBase< Arc > * base
void InitStateIterator(StateIteratorData< Arc > *data) const
StateId NumStates() const
const Arc & Value() const
static ConstFstImpl< Arc, Unsigned > * Read(std::istream &strm, const FstReadOptions &opts)
constexpr uint64 kMutable
Weight Final(StateId s) const
void InitStateIterator(StateIteratorData< Arc > *data) const override
bool AlignInput(std::istream &strm)
const Impl * GetImpl() const
virtual const SymbolTable * OutputSymbols() const =0