我有一个通过 Java 中的 Mallet 训练的 LDA 模型。Mallet LDA 模型生成了三个文件，这允许我从文件运行模型并推断新文本的主题分布。
现在我想实现一个 Python 工具，该工具能够基于经过训练的 LDA 模型推断给定新文本的主题分布。我不想在 Python 中重新训练 LDA 模型。因此，我想知道是否可以将经过训练的 Mallet LDA 模型加载到 Gensim 或任何其他 python LDA 包中。如果是这样，我该怎么做？
感谢您的任何回答或评论。
简而言之，是的，你可以！使用 mallet 的好处是，一旦运行，您就不必遍历并重新标记主题。我正在做一些非常类似的事情 - 我将在下面发布我的代码以及一些有用的链接。一旦您的模型经过训练，保存笔记本小部件状态，您就可以在具有相同主题分配的新的和不同的数据集上自由运行您的模型。该代码包括测试和验证集。确保您已经下载了 mallet 和 java，然后尝试以下操作：
\n\n# future bridges python 2 and 3\r\nfrom __future__ import print_function\r\n\r\n# pandas works with data structures, data manipulation, and analysis specifically for numerical tables, and series like \r\n# the csv we are using here today\r\nimport pandas as pd\r\n\r\nfrom sklearn import datasets, linear_model\r\nfrom sklearn.model_selection import train_test_split\r\nfrom matplotlib import pyplot as plt\r\n\r\n# Gensim unsupervised topic modeling, natural language processing, statistical machine learning\r\nimport gensim\r\n# convert a document to a list of tolkens\r\nfrom gensim.utils import simple_preprocess\r\n# remove stopwords - words that are not telling: "it" "I" "the" "and" ect.\r\nfrom gensim.parsing.preprocessing import STOPWORDS\r\n# corpus iterator \r\nfrom gensim import corpora, models\r\n\r\n# nltk - Natural Language Toolkit\r\n# lemmatized\xe2\x80\x8a\xe2\x80\x94\xe2\x80\x8awords in third person are changed to first person and verbs in past and future tenses are changed \r\n# into present.\r\n# stemmed\xe2\x80\x8a\xe2\x80\x94\xe2\x80\x8awords are reduced to their root form.\r\nimport nltk\r\nnltk.download(\'wordnet\')\r\nfrom nltk.stem import WordNetLemmatizer, SnowballStemmer\r\nfrom nltk.stem.porter import *\r\n\r\n# NumPy - multidimensional arrays, matrices, and high-level mathematical formulas\r\nimport numpy as np\r\nnp.random.seed(2018)\r\n\r\nimport os\r\nfrom gensim.models.wrappers import LdaMallet\r\nfrom pathlib import Path\r\nimport codecs\r\nimport logging\r\n\r\nimport re\r\nimport numpy as np\r\nimport pandas as pd\r\nfrom pprint import pprint\r\n\r\n# Gensim\r\nimport gensim\r\nimport gensim.corpora as corpora\r\nfrom gensim.utils import simple_preprocess\r\nfrom gensim.models import CoherenceModel\r\n\r\n# spacy for lemmatization\r\nimport spacy\r\n\r\n# Plotting tools\r\nimport pyLDAvis\r\nimport pyLDAvis.gensim # don\'t skip this\r\nimport matplotlib.pyplot as plt\r\n%matplotlib inline\r\n\r\n# Enable logging for gensim - optional\r\nimport logging\r\nlogging.basicConfig(format=\'%(asctime)s : %(levelname)s : %(message)s\', level=logging.ERROR)\r\n\r\nimport warnings\r\nwarnings.filterwarnings("ignore",category=DeprecationWarning)\r\n\r\nlogging.basicConfig(format="%(asctime)s : %(levelname)s : %(message)s", level=logging.INFO)\r\n\r\ndata = pd.read_csv(\'YourData.csv\', encoding = "ISO-8859-1");\r\ndata_text = data[[\'Preprocessed Document or your comments column title\']]\r\ndata_text[\'index\'] = data_text.index\r\ndocuments = data_text\r\n\r\n# Create functions to lemmatize stem, and preprocess\r\n\r\n# turn beautiful, beautifuly, beautified into stem beauti \r\ndef lemmatize_stemming(text):\r\n stemmer = PorterStemmer()\r\n return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos=\'v\'))\r\n\r\n# parse docs into individual words ignoring words that are less than 3 letters long\r\n# and stopwords: him, her, them, for, there, ect since "their" is not a topic.\r\n# then append the tolkens into a list\r\ndef preprocess(text):\r\n result = []\r\n for token in gensim.utils.simple_preprocess(text):\r\n newStopWords = [\'yourStopWord1\', \'yourStopWord2\']\r\n if token not in gensim.parsing.preprocessing.STOPWORDS and token not in newStopWords and len(token) > 3:\r\n nltk.bigrams(token)\r\n result.append(lemmatize_stemming(token))\r\n return result\r\n\r\n# gensim.parsing.preprocessing.STOPWORDS\r\n\r\n# look at a random row 4310 and see if things worked out\r\n# note that the document created was already preprocessed\r\n\r\ndoc_sample = documents[documents[\'index\'] == 4310].values[0][0]\r\nprint(\'original document: \')\r\nwords = []\r\nfor word in doc_sample.split(\' \'):\r\n words.append(word)\r\nprint(words)\r\nprint(\'\\n\\n tokenized and lemmatized document: \')\r\nprint(preprocess(doc_sample))\r\n\r\n# let\xe2\x80\x99s look at ten rows passed through the lemmatize stemming and preprocess\r\n\r\ndocuments = documents.dropna(subset=[\'Preprocessed Document\'])\r\nprocessed_docs = documents[\'Preprocessed Document\'].map(preprocess)\r\nprocessed_docs[:10]\r\n\r\n# we create a dictionary of all the words in the csv by iterating through\r\n# contains the number of times a word appears in the training set.\r\n\r\ndictionary_valid = gensim.corpora.Dictionary(processed_docs[20000:])\r\ncount = 0\r\nfor k, v in dictionary_valid.iteritems():\r\n print(k, v)\r\n count += 1\r\n if count > 30:\r\n break\r\n \r\n # we create a dictionary of all the words in the csv by iterating through\r\n# contains the number of times a word appears in the training set.\r\n\r\ndictionary_test = gensim.corpora.Dictionary(processed_docs[:20000])\r\ncount = 0\r\nfor k, v in dictionary_test.iteritems():\r\n print(k, v)\r\n count += 1\r\n if count > 30:\r\n break\r\n \r\n# we want to throw out words that are so frequent that they tell us little about the topic \r\n# as well as words that are too infrequent >15 rows then keep just 100,000 words\r\n\r\ndictionary_valid.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)\r\n\r\n# we want to throw out words that are so frequent that they tell us little about the topic \r\n# as well as words that are too infrequent >15 rows then keep just 100,000 words\r\n\r\ndictionary_test.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)\r\n\r\n# the words become numbers and are then counted for frequency\r\n# consider a random row 4310 - it has 8 words word indexed 2 shows up once\r\n# preview the bag of words\r\n\r\nbow_corpus_valid = [dictionary_valid.doc2bow(doc) for doc in processed_docs]\r\nbow_corpus_valid[4310]\r\n\r\n# the words become numbers and are then counted for frequency\r\n# consider a random row 4310 - it has 8 words word indexed 2 shows up once\r\n# preview the bag of words\r\n\r\nbow_corpus_test = [dictionary_test.doc2bow(doc) for doc in processed_docs]\r\nbow_corpus_test[4310]\r\n\r\n# same thing in more words\r\n\r\nbow_doc_4310 = bow_corpus_test[4310]\r\nfor i in range(len(bow_doc_4310)):\r\n print("Word {} (\\"{}\\") appears {} time.".format(bow_doc_4310[i][0], \r\n dictionary_test[bow_doc_4310[i][0]], \r\nbow_doc_4310[i][1]))\r\n\r\nmallet_path = \'C:/mallet/mallet-2.0.8/bin/mallet.bat\'\r\n\r\nldamallet_test = gensim.models.wrappers.LdaMallet(mallet_path, corpus=bow_corpus_test, num_topics=20, id2word=dictionary_test)\r\n\r\nresult = (ldamallet_test.show_topics(num_topics=20, num_words=10,formatted=False))\r\nfor each in result:\r\n print (each)\r\n \r\nmallet_path = \'C:/mallet/mallet-2.0.8/bin/mallet.bat\'\r\n\r\nldamallet_valid = gensim.models.wrappers.LdaMallet(mallet_path, corpus=bow_corpus_valid, num_topics=20, id2word=dictionary_valid)\r\n\r\nresult = (ldamallet_valid.show_topics(num_topics=20, num_words=10,formatted=False))\r\nfor each in result:\r\n print (each)\r\n \r\n# Show Topics\r\nfor idx, topic in ldamallet_test.print_topics(-1):\r\n print(\'Topic: {} \\nWords: {}\'.format(idx, topic))\r\n \r\n# Show Topics\r\nfor idx, topic in ldamallet_valid.print_topics(-1):\r\n print(\'Topic: {} \\nWords: {}\'.format(idx, topic))\r\n \r\n# check out the topics - 30 words - 20 topics\r\n\r\nldamallet_valid.print_topics(idx, 30)\r\n\r\n# check out the topics - 30 words - 20 topics\r\n\r\nldamallet_test.print_topics(idx, 30)\r\n\r\n# Compute Coherence Score\r\ncoherence_model_ldamallet_valid = CoherenceModel(model=ldamallet_valid, texts=processed_docs, dictionary=dictionary_valid, coherence=\'c_v\')\r\ncoherence_ldamallet_valid = coherence_model_ldamallet_valid.get_coherence()\r\nprint(\'\\nCoherence Score: \', coherence_ldamallet_valid)\r\n\r\n# Compute Coherence Score\r\ncoherence_model_ldamallet_test = CoherenceModel(model=ldamallet_test, texts=processed_docs, dictionary=dictionary_test, coherence=\'c_v\')\r\ncoherence_ldamallet_test = coherence_model_ldamallet_test.get_coherence()\r\nprint(\'\\nCoherence Score: \', coherence_ldamallet_test)看看 16： https: //www.machinelearningplus.com/nlp/topic-modeling-gensim-python/ \n这有帮助：https://rare-technologies.com/tutorial-on-mallet-in-python/ \nand这个：https: //radimrehurek.com/gensim/models/wrappers/ldamallet.html
\n\n我希望这个帮助能祝你好运 ：）
\n| 归档时间： |
|
| 查看次数： |
1045 次 |
| 最近记录： |