#!/usr/bin/env bash

set -x

NWC_TOOLKIT_BIN=${HOME}/nwc-toolkit/build
NEOLOGD_DIR=/usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd

${NWC_TOOLKIT_BIN}/nwc-toolkit-unicode-normalizer ja.txt.xz | \
	${NWC_TOOLKIT_BIN}/nwc-toolkit-text-filter | \
	mecab -F "%m\t%f[7]\t" -U "<UNK>\t<UNK>\t" -E "\n" -d ${NEOLOGD_DIR} | \
	./src/convert_yomi | gzip -c > ja.mecab-neologd.txt.gz

${NWC_TOOLKIT_BIN}/nwc-toolkit-ngram-counter -n 3 -l 12288 -bs -p ngms-temp \
	ja.mecab-neologd.txt.gz

${NWC_TOOLKIT_BIN}/nwc-toolkit-ngram-merger ngms-temp*.gz -o ngms.gz

rm -f ngms-temp*.gz