35 "Single non-white-space character delimiter inside sequences of " 36 "feature symbols and output symbols");
38 "Special symbol that designates an empty sequence");
40 DEFINE_string(start_symbol,
"<s>",
"Start of sentence symbol");
44 "Treat input model as a classifier instead of a tagger");
50 return FST_FLAGS_delimiter.size() == 1 &&
51 !std::isspace(FST_FLAGS_delimiter[0]);
55 bool okay = !FST_FLAGS_empty_symbol.empty();
56 for (
size_t i = 0; i < FST_FLAGS_empty_symbol.size(); ++i) {
57 char c = FST_FLAGS_empty_symbol[i];
58 if (std::isspace(c)) okay =
false;
64 const std::string &epsilon_symbol,
65 const std::string &unknown_symbol,
const std::string &vocab,
66 char **models,
int models_len,
const std::string &out,
67 const std::string &save_isymbols,
68 const std::string &save_fsymbols,
69 const std::string &save_osymbols) {
71 models_len, out, save_isymbols, save_fsymbols,
73 Apply<Operation<LinearCompileArgs>>(
"LinearCompileTpl", arc_type, &args);
80 std::istringstream strm(str);
82 while (strm >> buf) out->push_back(buf);
86 std::set<std::string, std::less<>> preds;
87 for (
int i = 0; i < models_len; ++i) {
88 std::ifstream in(models[i]);
89 if (!in)
LOG(FATAL) <<
"Failed to open " << models[i];
91 std::getline(in, line);
93 while (std::getline(in, line)) {
95 std::vector<std::string> fields;
97 if (fields.size() != 3)
98 LOG(FATAL) <<
"Wrong number of fields in source " << models[i]
99 <<
", line " << num_line;
100 preds.insert(fields[1]);
void SplitByWhitespace(const std::string &str, std::vector< std::string > *out)
REGISTER_FST_OPERATION_3ARCS(Compress, CompressArgs)
std::tuple< const std::string &, const std::string &, const std::string &, char **, int, const std::string &, const std::string &, const std::string &, const std::string & > LinearCompileArgs
DEFINE_string(delimiter,"|","Single non-white-space character delimiter inside sequences of ""feature symbols and output symbols")
bool ValidateEmptySymbol()
int ScanNumClasses(char **models, int models_len)
void LinearCompile(const std::string &arc_type, const std::string &epsilon_symbol, const std::string &unknown_symbol, const std::string &vocab, char **models, int models_len, const std::string &out, const std::string &save_isymbols, const std::string &save_fsymbols, const std::string &save_osymbols)
DEFINE_bool(classifier, false,"Treat input model as a classifier instead of a tagger")
void LinearCompileTpl(LinearCompileArgs *args)