.NET下文本相似度算法余弦定理和SimHash浅析及应用(3)

private void GenerateTermWeight()
        {           
            for(int i=0; i < _numTerms   ; i++)
            {
                for(int j=0; j < _numDocs ; j++)               
                    _termWeight[i][j]=ComputeTermWeight (i, j);
            }
        }
 
        private float GetTermFrequency(int term, int doc)
        {           
            int freq=_termFreq [term][doc];
            int maxfreq=_maxTermFreq[doc];           
           
            return ( (float) freq/(float)maxfreq );
        }
 
        private float GetInverseDocumentFrequency(int term)
        {
            int df=_docFreq[term];
            return Log((float) (_numDocs) / (float) df );
        }
 
        private float ComputeTermWeight(int term, int doc)
        {
            float tf=GetTermFrequency (term, doc);
            float idf=GetInverseDocumentFrequency(term);
            return tf * idf;
        }
       
        private  float[] GetTermVector(int doc)
        {
            float[] w=new float[_numTerms] ;
            for (int i=0; i < _numTerms; i++)
                w[i]=_termWeight[i][doc];
            return w;
        }
 
        public float GetSimilarity(int doc_i, int doc_j)
        {
            float[] vector1=GetTermVector (doc_i);
            float[] vector2=GetTermVector (doc_j);
            return TermVector.ComputeCosineSimilarity(vector1, vector2);
        }
       
        private IDictionary GetWordFrequency(string input)
        {
            string convertedInput=input.ToLower() ;
            Tokeniser tokenizer=new Tokeniser() ;
            String[] words=tokenizer.Partition(convertedInput);
            Array.Sort(words);
           
            String[] distinctWords=GetDistinctWords(words);
                       
            IDictionary result=new Hashtable();
            for (int i=0; i < distinctWords.Length; i++)
            {
                object tmp;
                tmp=CountWords(distinctWords[i], words);
                result[distinctWords[i]]=tmp;
            }
            return result;
        }               
               
        private string[] GetDistinctWords(String[] input)
        {               
            if (input == null)           
                return new string[0];           
            else
            {
                ArrayList list=new ArrayList() ;
               
                for (int i=0; i < input.Length; i++)
                    if (!list.Contains(input[i])) // N-GRAM SIMILARITY?
                        list.Add(input[i]);
                return Tokeniser.ArrayListToArray(list) ;
            }
        }