Python机器学习NLP自然语言处理基本操作精确分词

目录

• 概述
• 分词器 jieba
• 安装
• 精确分词
• 全模式
• 搜索引擎模式
• 获取词性

pip install jieba

import jieba
print(jieba.__version__)

jieba.cut(content, cut_all=False)

import jieba
# 定义文本
content = "自然语言处理是人工智能和语言学领域的分支学科。此领域探讨如何处理及运用自然语言；自然语言处理包括多方面和步骤，基本有认知、理解、生成等部分。"
# 精确分词
seg = jieba.cut(content, cut_all=False)
# 调试输出
print([word for word in seg])

Building prefix dict from the default dictionary ...
Loading model from cache C:UsersWindowsAppDataLocalTempjieba.cache
Loading model cost 0.984 seconds.
Prefix dict has been built successfully.
["自然语言", "处理", "是", "人工智能", "和", "语言学", "领域", "的", "分支", "学科", "。", "此", "领域", "探讨", "如何", "处理", "及", "运用", "自然语言", "；", "自然语言", "处理", "包括", "多方面", "和", "步骤", "，", "基本", "有", "认知", "、", "理解", "、", "生成", "等", "部分", "。"]

C:UsersWindowsAnaconda3pythonw.exe "C:/Users/Windows/Desktop/project/NLP 基础/结巴.py"
Building prefix dict from the default dictionary ...
Loading model from cache C:UsersWindowsAppDataLocalTempjieba.cache
["自然", "自然语言", "语言", "处理", "是", "人工", "人工智能", "智能", "和", "语言", "语言学", "领域", "的", "分支", "学科", "。", "此", "领域", "探讨", "如何", "何处", "处理", "及", "运用", "自然", "自然语言", "语言", "；", "自然", "自然语言", "语言", "处理", "包括", "多方", "多方面", "方面", "和", "步骤", "，", "基本", "有", "认知", "、", "理解", "、", "生成", "等", "部分", "。"]
Loading model cost 0.999 seconds.
Prefix dict has been built successfully.

Building prefix dict from the default dictionary ...
Loading model from cache C:UsersWindowsAppDataLocalTempjieba.cache
["自然", "自然语言", "语言", "处理", "是", "人工", "人工智能", "智能", "和", "语言", "语言学", "领域", "的", "分支", "学科", "。", "此", "领域", "探讨", "如何", "何处", "处理", "及", "运用", "自然", "自然语言", "语言", "；", "自然", "自然语言", "语言", "处理", "包括", "多方", "多方面", "方面", "和", "步骤", "，", "基本", "有", "认知", "、", "理解", "、", "生成", "等", "部分", "。"]
Loading model cost 0.999 seconds.
Prefix dict has been built successfully.

import jieba
# 定义文本
content = "自然语言处理是人工智能和语言学领域的分支学科。此领域探讨如何处理及运用自然语言；自然语言处理包括多方面和步骤，基本有认知、理解、生成等部分。"
# 搜索引擎模式
seg = jieba.cut_for_search(content)
# 调试输出
print([word for word in seg])

Building prefix dict from the default dictionary ...
Loading model from cache C:UsersWindowsAppDataLocalTempjieba.cache
[("自然语言", "l"), ("处理", "v"), ("是", "v"), ("人工智能", "n"), ("和", "c"), ("语言学", "n"), ("领域", "n"), ("的", "uj"), ("分支", "n"), ("学科", "n"), ("。", "x"), ("此", "zg"), ("领域", "n"), ("探讨", "v"), ("如何", "r"), ("处理", "v"), ("及", "c"), ("运用", "vn"), ("自然语言", "l"), ("；", "x"), ("自然语言", "l"), ("处理", "v"), ("包括", "v"), ("多方面", "m"), ("和", "c"), ("步骤", "n"), ("，", "x"), ("基本", "n"), ("有", "v"), ("认知", "v"), ("、", "x"), ("理解", "v"), ("、", "x"), ("生成", "v"), ("等", "u"), ("部分", "n"), ("。", "x")]
Loading model cost 1.500 seconds.
Prefix dict has been built successfully.

import jieba.posseg as psg
# 定义文本
content = "自然语言处理是人工智能和语言学领域的分支学科。此领域探讨如何处理及运用自然语言；自然语言处理包括多方面和步骤，基本有认知、理解、生成等部分。"
# 分词
seg = psg.lcut(content)
# 获取词性
part_of_speech = [(x.word, x.flag) for x in seg]
# 调试输出
print(part_of_speech)

Building prefix dict from the default dictionary ...
Loading model from cache C:UsersWindowsAppDataLocalTempjieba.cache
[("自然语言", "l"), ("处理", "v"), ("是", "v"), ("人工智能", "n"), ("和", "c"), ("语言学", "n"), ("领域", "n"), ("的", "uj"), ("分支", "n"), ("学科", "n"), ("。", "x"), ("此", "zg"), ("领域", "n"), ("探讨", "v"), ("如何", "r"), ("处理", "v"), ("及", "c"), ("运用", "vn"), ("自然语言", "l"), ("；", "x"), ("自然语言", "l"), ("处理", "v"), ("包括", "v"), ("多方面", "m"), ("和", "c"), ("步骤", "n"), ("，", "x"), ("基本", "n"), ("有", "v"), ("认知", "v"), ("、", "x"), ("理解", "v"), ("、", "x"), ("生成", "v"), ("等", "u"), ("部分", "n"), ("。", "x")]
Loading model cost 1.500 seconds.
Prefix dict has been built successfully.