Nei*_*man 3 php algorithm function
我正在努力寻找/创建一个可以确定随机5个字母组合的可读性的算法.
到目前为止,我发现的最接近的是来自这个3岁的StackOverflow线程:
<?php
// Score: 1
echo pronounceability('namelet') . "\n";
// Score: 0.71428571428571
echo pronounceability('nameoic') . "\n";
function pronounceability($word) {
static $vowels = array
(
'a',
'e',
'i',
'o',
'u',
'y'
);
static $composites = array
(
'mm',
'll',
'th',
'ing'
);
if (!is_string($word)) return false;
// Remove non letters and put in lowercase
$word = preg_replace('/[^a-z]/i', '', $word);
$word = strtolower($word);
// Special case
if ($word == 'a') return 1;
$len = strlen($word);
// Let's not parse an empty string
if ($len == 0) return 0;
$score = 0;
$pos = 0;
while ($pos < $len) {
// Check if is allowed composites
foreach ($composites as $comp) {
$complen = strlen($comp);
if (($pos + $complen) < $len) {
$check = substr($word, $pos, $complen);
if ($check == $comp) {
$score += $complen;
$pos += $complen;
continue 2;
}
}
}
// Is it a vowel? If so, check if previous wasn't a vowel too.
if (in_array($word[$pos], $vowels)) {
if (($pos - 1) >= 0 && !in_array($word[$pos - 1], $vowels)) {
$score += 1;
$pos += 1;
continue;
}
} else { // Not a vowel, check if next one is, or if is end of word
if (($pos + 1) < $len && in_array($word[$pos + 1], $vowels)) {
$score += 2;
$pos += 2;
continue;
} elseif (($pos + 1) == $len) {
$score += 1;
break;
}
}
$pos += 1;
}
return $score / $len;
}
?>
......但它远非完美,给出了一些相当奇怪的误报:
使用此功能,所有以下速率均可发音,(7/10以上)
可能比我聪明的人调整这个算法可能是这样的:
(我已经做了大量的研究/谷歌搜索,这似乎是每个人在过去3年中一直在引用/使用的主要发音功能,所以我确信更新,更精致的版本将受到赞赏更广泛的社区,而不仅仅是我!).
根据关于"在字母上使用马尔可夫模型"的链接问题的建议
使用马尔可夫模型(当然是字母,而不是文字).单词的概率是发音容易度的一个很好的代理.
我想我会尝试一下并取得一些成功.
我将一个真正的5个字母的单词列表复制到一个文件中作为我的数据集(这里 ......嗯,实际上在这里).
然后我使用隐马尔可夫模型(基于One-gram,Bi-gram和Tri-gram)来预测目标词在该数据集中出现的可能性.
(通过某种语音转录作为其中一个步骤,可以获得更好的结果.)
首先,我计算数据集中字符序列的概率.
例如,如果"A"出现50次,并且数据集中只有250个字符,则"A"的概率为50/250或.2.
为bigrams'AB','AC'做同样的事......
为三卦'ABC','ABD'做同样的事......
基本上,我对"ABCDE"这个词的得分是由:
您可以将所有这些相乘以获得目标单词出现在数据集中的估计概率(但这非常小).
因此,我们将每个日志记录下来并将它们加在一起.
现在我们有一个分数来估计我们的目标词在数据集中出现的可能性.
我编码这是C#,并发现大于负160的分数非常好.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace Pronouncability
{
class Program
{
public static char[] alphabet = new char[]{ 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z' };
public static List<string> wordList = loadWordList(); //Dataset of 5-letter words
public static Random rand = new Random();
public const double SCORE_LIMIT = -160.00;
/// <summary>
/// Generates random words, until 100 of them are better than
/// the SCORE_LIMIT based on a statistical score.
/// </summary>
public static void Main(string[] args)
{
Dictionary<Tuple<char, char, char>, int> trigramCounts = new Dictionary<Tuple<char, char, char>, int>();
Dictionary<Tuple<char, char>, int> bigramCounts = new Dictionary<Tuple<char, char>, int>();
Dictionary<char, int> onegramCounts = new Dictionary<char, int>();
calculateProbabilities(onegramCounts, bigramCounts, trigramCounts);
double totalTrigrams = (double)trigramCounts.Values.Sum();
double totalBigrams = (double)bigramCounts.Values.Sum();
double totalOnegrams = (double)onegramCounts.Values.Sum();
SortedList<double, string> randomWordsScores = new SortedList<double, string>();
while( randomWordsScores.Count < 100 )
{
string randStr = getRandomWord();
if (!randomWordsScores.ContainsValue(randStr))
{
double score = getLikelyhood(randStr,trigramCounts, bigramCounts, onegramCounts, totalTrigrams, totalBigrams, totalOnegrams);
if (score > SCORE_LIMIT)
{
randomWordsScores.Add(score, randStr);
}
}
}
//Right now randomWordsScores contains 100 random words which have
//a better score than the SCORE_LIMIT, sorted from worst to best.
}
/// <summary>
/// Generates a random 5-letter word
/// </summary>
public static string getRandomWord()
{
char c0 = (char)rand.Next(65, 90);
char c1 = (char)rand.Next(65, 90);
char c2 = (char)rand.Next(65, 90);
char c3 = (char)rand.Next(65, 90);
char c4 = (char)rand.Next(65, 90);
return "" + c0 + c1 + c2 + c3 + c4;
}
/// <summary>
/// Returns a score for how likely a given word is, based on given trigrams, bigrams, and one-grams
/// </summary>
public static double getLikelyhood(string wordToScore, Dictionary<Tuple<char, char,char>, int> trigramCounts, Dictionary<Tuple<char, char>, int> bigramCounts, Dictionary<char, int> onegramCounts, double totalTrigrams, double totalBigrams, double totalOnegrams)
{
wordToScore = wordToScore.ToUpper();
char[] letters = wordToScore.ToCharArray();
Tuple<char, char>[] bigrams = new Tuple<char, char>[]{
new Tuple<char,char>( wordToScore[0], wordToScore[1] ),
new Tuple<char,char>( wordToScore[1], wordToScore[2] ),
new Tuple<char,char>( wordToScore[2], wordToScore[3] ),
new Tuple<char,char>( wordToScore[3], wordToScore[4] )
};
Tuple<char, char, char>[] trigrams = new Tuple<char, char, char>[]{
new Tuple<char,char,char>( wordToScore[0], wordToScore[1], wordToScore[2] ),
new Tuple<char,char,char>( wordToScore[1], wordToScore[2], wordToScore[3] ),
new Tuple<char,char,char>( wordToScore[2], wordToScore[3], wordToScore[4] ),
};
double score = 0;
foreach (char c in letters)
{
score += Math.Log((((double)onegramCounts[c]) / totalOnegrams));
}
foreach (Tuple<char, char> pair in bigrams)
{
score += Math.Log((((double)bigramCounts[pair]) / totalBigrams));
}
foreach (Tuple<char, char, char> trio in trigrams)
{
score += 5.0*Math.Log((((double)trigramCounts[trio]) / totalTrigrams));
}
return score;
}
/// <summary>
/// Build the probability tables based on the dataset (WordList)
/// </summary>
public static void calculateProbabilities(Dictionary<char, int> onegramCounts, Dictionary<Tuple<char, char>, int> bigramCounts, Dictionary<Tuple<char, char, char>, int> trigramCounts)
{
foreach (char c1 in alphabet)
{
foreach (char c2 in alphabet)
{
foreach( char c3 in alphabet)
{
trigramCounts[new Tuple<char, char, char>(c1, c2, c3)] = 1;
}
}
}
foreach( char c1 in alphabet)
{
foreach( char c2 in alphabet)
{
bigramCounts[ new Tuple<char,char>(c1,c2) ] = 1;
}
}
foreach (char c1 in alphabet)
{
onegramCounts[c1] = 1;
}
foreach (string word in wordList)
{
for (int pos = 0; pos < 3; pos++)
{
trigramCounts[new Tuple<char, char, char>(word[pos], word[pos + 1], word[pos + 2])]++;
}
for (int pos = 0; pos < 4; pos++)
{
bigramCounts[new Tuple<char, char>(word[pos], word[pos + 1])]++;
}
for (int pos = 0; pos < 5; pos++)
{
onegramCounts[word[pos]]++;
}
}
}
/// <summary>
/// Get the dataset (WordList) from file.
/// </summary>
public static List<string> loadWordList()
{
string filePath = "WordList.txt";
string text = File.ReadAllText(filePath);
List<string> result = text.Split(' ').ToList();
return result;
}
}
}
在我的例子中,我将三元组概率缩放5.
我还在所有计数中加一,所以我们不会乘以零.
我不是一个php程序员,但这个技术很容易实现.
使用一些缩放因子,尝试不同的数据集,或添加一些其他检查,如上面建议的那样.
| 归档时间： |
|
| 查看次数： |
2360 次 |
| 最近记录： |