Skip to content

Commit

Permalink
Verified inconsistent tokenization against built-in Lucene Kuromoji
Browse files Browse the repository at this point in the history
  • Loading branch information
@azagniotov
azagniotov committed Feb 13, 2024
1 parent 92bf507 commit 0d42bee
Showing 1 changed file with 22 additions and 20 deletions.
42 changes: 22 additions & 20 deletions .../java/io/github/azagniotov/lucene/analysis/ja/sudachi/tokenizer/SudachiTokenizerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public static Object[][] querySurfaces() {
{"機能性食品", new Object[] {"機能", "性", "食品"}},
{"労働者協同組合", new Object[] {"労働", "者", "協同", "組合"}},
{"客室乗務員", new Object[] {"客室", "乗務", "員"}},
// MeCab IPAdic: 医薬品, 安全, 管理, 責任, 者
// Built-in Lucene's Kuromoji: 医薬品, 安全, 管理, 責任, 者
{"医薬品安全管理責任者", new Object[] {"医薬", "品", "安全", "管理", "責任", "者"}},
{
"I was playing on the computers all day",
Expand All @@ -98,65 +98,67 @@ public static Object[][] querySurfaces() {
},
{"消費者安全調査委員会", new Object[] {"消費", "者", "安全", "調査", "委員", "会"}},
{"選挙管理委員会", new Object[] {"選挙", "管理", "委員", "会"}},
// MeCab IPAdic: カンヌ国際映画祭
// Built-in Lucene's Kuromoji:: カンヌ国際映画祭
{"カンヌ国際映画祭", new Object[] {"カンヌ", "国際", "映画", "祭"}},
// MeCab IPAdic: さ, っぽ, ろ, テレビ塔
// Built-in Lucene's Kuromoji: さ, っぽ, ろ, テレビ塔
{"さっぽろテレビ塔", new Object[] {"さっぽろ", "テレビ", "塔"}},
{"京都。東京.東京都。京都", new Object[] {"京都", "東京", "東京", "都", "京都"}},
{"清水寺は東京都にあります", new Object[] {"清水寺", "は", "東京", "都", "に", "あり", "ます"}},
// MeCab IPAdic: ちい, か, わ
// Built-in Lucene's Kuromoji: ちい, か, わ
{"ちいかわ", new Object[] {"ちいかわ"}},
{"くら寿司ちいかわ", new Object[] {"くら", "寿司", "ちいかわ"}},
{"ソフトウェアエンジニア", new Object[] {"ソフトウェア", "エンジニア"}},
// MeCab IPAdic: 関西国際空港
{"関西国際空港", new Object[] {"関西", "国際", "空港"}},
{"五十嵐淳子", new Object[] {"五十嵐", "淳子"}},
{"鬼滅", new Object[] {"鬼", "滅"}},
// MeCab IPAdic: 鬼, 滅, の, 刃
// Built-in Lucene's Kuromoji: 図書館
{"図書館", new Object[] {"図書", "館"}},
{"ｺｰﾋｰ ﾀｸｼｰ", new Object[] {"ｺｰﾋｰ", "ﾀｸｼｰ"}},

// Built-in Lucene's Kuromoji: 鬼, 滅, の, 刃
{"鬼滅の刃", new Object[] {"鬼滅の刃"}}, // Kimetsu no Yaiba
// MeCab IPAdic: 呪術, 廻, 戦
// Built-in Lucene's Kuromoji: 呪術, 廻, 戦
{"呪術廻戦", new Object[] {"呪術廻戦"}}, // Jujutsu Kaizen
// MeCab IPAdic: お, 試し, 用, 使い切り
// Built-in Lucene's Kuromoji: お, 試し, 用, 使い切り
{"お試し用(使い切り)", new Object[] {"お", "試し", "用", "使い", "切り"}},
// MeCab IPAdic: 聖川, 真, 斗
// Built-in Lucene's Kuromoji: 聖川, 真, 斗
{"聖川真斗", new Object[] {"聖川", "真斗"}},
{"IKEAの椅子", new Object[] {"IKEA", "の", "椅子"}},
{"京都東部", new Object[] {"京都", "東部"}},
{"水 / 化粧+水", new Object[] {"水", "化粧", "水"}},
{"平成", new Object[] {"平成"}},
{"季節感···冬", new Object[] {"季節", "感", "冬"}},
// MeCab IPAdic: ユニクロポロシャツ
// Built-in Lucene's Kuromoji: ユニ, クロ, ポロシャツ
{"ユニクロポロシャツ", new Object[] {"ユニクロ", "ポロシャツ"}},
{"アンパスィ", new Object[] {"アンパスィ"}},
// MeCab IPAdic: トラックボール
// Built-in Lucene's Kuromoji: トラックボール
{"トラックボール", new Object[] {"トラック", "ボール"}},
{"日本限定ソファ", new Object[] {"日本", "限定", "ソファ"}},
{"吾輩は猫である", new Object[] {"吾輩", "は", "猫", "で", "ある"}},
{"あなたが誰かを殺した", new Object[] {"あなた", "が", "誰", "か", "を", "殺し", "た"}},
// MeCab IPAdic: 日本語, 単, 話, 版
// Built-in Lucene's Kuromoji: 日本語, 単, 話, 版
{"日本語【単話版】", new Object[] {"日本", "語", "単", "話", "版"}},
// MeCab IPAdic: 日本語, 単, 話, 版
// Built-in Lucene's Kuromoji: 日本語, 単, 話, 版
{"日本語 単話版", new Object[] {"日本", "語", "単", "話", "版"}},
{"アフラ・マヅダ", new Object[] {"アフラ・マヅダ"}},
{"楷・行書", new Object[] {"楷・行書"}},
// MeCab IPAdic: 楷行, 書
// Built-in Lucene's Kuromoji: 楷, 行書
{"楷行書", new Object[] {"楷行書"}},
{"日本語と「記号」の話し", new Object[] {"日本", "語", "と", "記号", "の", "話し"}},
{"桃太郎電鉄", new Object[] {"桃太郎", "電鉄"}},
{"甲斐田晴", new Object[] {"甲斐田", "晴"}},
{"椎名ニキ", new Object[] {"椎名", "ニキ"}},
{"シルバニア", new Object[] {"シルバニア"}},
{"ポケカ", new Object[] {"ポケカ"}},
// MeCab IPAdic: おそ, 松
// Built-in Lucene's Kuromoji: おそ, 松
{"おそ松", new Object[] {"おそ松"}},
// MeCab IPAdic: 日本経済新聞
{"日本経済新聞", new Object[] {"日本", "経済", "新聞"}},
// MeCab IPAdic: りん, ご, ジュース, を, 飲ん, だ
// Built-in Lucene's Kuromoji: りん, ご, ジュース, を, 飲ん, だ
{"りんごジュースを飲んだ", new Object[] {"りんご", "ジュース", "を", "飲ん", "だ"}},
// MeCab IPAdic: 日本人
// Built-in Lucene's Kuromoji: 日本人
{"日本人", new Object[] {"日本", "人"}},
{"セカオワ", new Object[] {"セカオワ"}},
// MeCab IPAdic: 魔法使い, の, 嫁
// Built-in Lucene's Kuromoji: 魔法使い, の, 嫁
{"魔法使いの嫁", new Object[] {"魔法", "使い", "の", "嫁"}},
{"ステラ ルー", new Object[] {"ステラ", "ルー"}},
{"ステラルー", new Object[] {"ステラルー"}},
Expand All @@ -167,7 +169,7 @@ public static Object[][] querySurfaces() {
{"神宮寺レン", new Object[] {"神宮寺", "レン"}},
{"ストウブ", new Object[] {"ストウブ"}},
{"マキタ", new Object[] {"マキタ"}},
// MeCab IPAdic: クロ, ミ
// Built-in Lucene's Kuromoji: クロ, ミ
{"クロミ", new Object[] {"クロミ"}},
// {"すもももももももものうち", new Object[] {"すもも", "も", "もも", "も", "もも", "の", "うち"}},
{"イーブイヒーローズbox未開封シュリンク", new Object[] {"イーブイヒーローズ", "box", "未", "開封", "シュリンク"}},
Expand Down

0 comments on commit 0d42bee

Please sign in to comment.