Merge pull request #72 from fujiki-1emon/dev/japanese

dev japanese

Author: fujiki-1emon

configs/japanese_job_v2_oscar2323_0-100.yaml

@@ -0,0 +1,21 @@
+base_dir: 'PROJECT_DIR/datasets/'
+targets:
+  - "oscar2323_0-100"
+output_dir: 'PROJECT_DIR/datasets/oscar2323_0-100_QFv2'
+
+n_dist: 128
+n_output: 1
+is_cluster: True
+is_local: False
+
+use_column: "content"
+min_doc_len: 50
+max_doc_len: 100000
+min_mean_word_len: 1
+max_mean_word_len: 10
+symbol_to_word_ratio: 0.1
+bullet_point_ratio: 0.9
+ellipsis_ratio:  0.3
+japanese_word_ratio: 0.8
+freq_char_cnt: 1
+separator_ratio: 0.1

requirements-ja.txt

@@ -5,4 +5,5 @@ bs4
 html2text
 python-stdnum
 numpy
-SudachiPy==0.5.4
+SudachiPy==0.5.4
+SudachiDict-core