问题描述
|
我有大量的（XXM-XXXM）字符串看起来像（一个小样本）：
我不知道所有可能的错误字符串，也不知道其排列。我想将所有类似的错误归为一组，并生成一些统计信息以显示每个错误字符串组的错误计数。
因此，从本质上讲，我想将最相似的字符串分组在一起，并且字符串可以属于多个组。
谢谢！
解决方法
免责声明：我从来没有解决过这样的问题。
我可以想到几种解决您的问题的方法：
您正在尝试将每一行聚类为一组聚类
检查数据挖掘算法
群集中每条线之间的差异较小，两个不同群集中的线之间的差异较大
您可以通过比较两条线的集合交点来快速收集相似的线：
set(line1.split) & set(line2.split)
import fileinput
CLUSTER_COUNT = 5
MAX_DISTANCE = 5
def main():
clusters = [Cluster() for i in range(CLUSTER_COUNT)]
MAXDISTANCE = 3
for line in fileinput.input():
words = set(line.split())
cluster = sorted(clusters,key=lambda c: c.distanceTo(words))[0]
cluster.addLine(words,line)
# print out results (FIXME: write clusters to separate files)
for cluster in clusters:
print \"CLUSTER:\",cluster.intersection
for line in cluster.lines:
print line
print \"-\" * 80
print
class Cluster(object):
def __init__(self):
self.intersection = set()
self.lines = []
def distanceTo(self,words):
if len(self.intersection) == 0:
return MAX_DISTANCE
return len(words) - len(self.intersection & words)
def addLine(self,words,line):
self.lines.append(line)
if len(self.intersection) == 0:
self.intersection = words
else:
self.intersection = self.intersection & words
if __name__ == \'__main__\':
main()
for bucket in buckets:
if re.match(bucket.regex,s):
bucket.matchingStrings.append(s)
break
# else if no buckets match,generate a new bucket/regex for s
pattern = r\"\\d+\"
re.sub(pattern,pattern,\"Failed to count from 0 to 600\")
# returns r\"Failed to count from \\d+ to \\d+\"
similars = get_similar_strings( input_string,max_distance );
for each similar in similars do
if is string then
//construct template from string
else
// increase count
done
map<string,int> mStringToInt;
struct structOneRecord
{
vector<int> vWords;
vector<vector<int>> vLines; // All the Lines under this record
vector<int> vIndexOfMutableWords;
bool operator < (const structOneRecord &n) const
{
if(vWords.size() != n.vWords.size())
return vWords.size() < n.vWords.size();
else
{
// Count diferences
vector<int> vCurrentIndexs;
for(int i=0; i<vWords.size(); i++)
{
if(vWords[i] != n.vWords[i])
vCurrentIndexs.push_back(i);
}
if(vCurrentIndexs.size() == 0)
return false;
int iHalf = vWords.size() / 2; // The diferences can\'t be bigger than hald the phrase
if(vCurrentIndexs.size() < iHalf)
{
if(vIndexOfMutableWords.size() == 0)
return false;
else
{
if(vIndexOfMutableWords.size() == vCurrentIndexs.size())
{
for(int i=0; i<vIndexOfMutableWords.size(); i++)
{
if(vIndexOfMutableWords[i] != vCurrentIndexs[i])
vWords[vCurrentIndexs[0]] < n.vWords[vCurrentIndexs[0]]; // Not equal
}
}
}
}
return vWords[vCurrentIndexs[0]] < n.vWords[vCurrentIndexs[0]];
}
}
};
vector<string> SplitString(const string &strInput,char cDelimiter,bool bSkipEmpty)
{
vector<string> vRetValue;
stringstream ss(strInput);
string strItem;
while(std::getline(ss,strItem,cDelimiter))
{
// Skip Empty
if(bSkipEmpty && strItem.size()==0)
continue;
vRetValue.push_back(strItem);
}
return vRetValue;
}
void main()
{
// To Test
vector<string> vInput;
vInput.push_back(\"Connection to 11.22.33.44 port 3940 timed out client 1.2.3.4 source port 3940\");
vInput.push_back(\"Error loading page somepage.php by client 2.3.4.5\");
vInput.push_back(\"Load of page someotherpage.php by client 2.3.4.8 failed due to error 4930\");
vInput.push_back(\"Connection to 11.22.33.55 port 3829 timed out client 1.2.3.6 source port 3944\");
vInput.push_back(\"Load of page alt.php by client 2.3.4.92 failed due to error 3829\");
vInput.push_back(\"Load of page alt2.php by client 2.3.4.95 failed due to error 3829\");
set<structOneRecord> sRecords;
for(int i=0; i<vInput.size(); i++)
{
vector<string> vWords = CMkDevStringUtilities::SplitString(vInput[i],\' \',true);
structOneRecord stRecord;
stRecord.vWords.resize(vWords.size());
for(int j=0; j<vWords.size(); j++)
{
map<string,int>::iterator mIte = mStringToInt.find(vWords[j]);
if(mIte == mStringToInt.end())
mIte = mStringToInt.insert(mStringToInt.begin(),make_pair(vWords[j],mStringToInt.size()));
stRecord.vWords[j] = mIte->second;
}
set<structOneRecord>::iterator sIte = sRecords.find(stRecord);
if(sIte != sRecords.end())
{
sIte->vLines.push_back(stRecord.vWords);
if(sIte->vIndexOfMutableWords.size() == 0)
{
// Count diferences
vector<int> vCurrentIndexs;
for(int i=0; i<stRecord.vWords.size(); i++)
{
if(sIte->vWords[i] != stRecord.vWords[i])
vCurrentIndexs.push_back(i);
}
sIte->vIndexOfMutableWords = vCurrentIndexs;
}
}
else
{
stRecord.vLines.push_back(stRecord.vWords);
sRecords.insert(stRecord);
}
}
}