From fc94b19c14cd44223b205469ed794abf08d9dfdb Mon Sep 17 00:00:00 2001 From: Guillaume Horel Date: Sat, 7 Nov 2015 18:49:40 -0500 Subject: initial tf-idf construction --- tf-idf.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tf-idf.py (limited to 'tf-idf.py') diff --git a/tf-idf.py b/tf-idf.py new file mode 100644 index 0000000..60280b8 --- /dev/null +++ b/tf-idf.py @@ -0,0 +1,33 @@ +import nltk +import string +import os + +from sklearn.feature_extraction.text import TfidfVectorizer +from nltk.stem.porter import PorterStemmer + +def stem_tokens(tokens, stemmer): + stemmed = [] + for item in tokens: + stemmed.append(stemmer.stem(item)) + return stemmed + +def tokenize(text): + tokens = nltk.word_tokenize(text) + stems = stem_tokens(tokens, stemmer) + return stems + +if __name__=="__main__": + token_dict = {} + stemmer = PorterStemmer() + for subdir, dirs, files in os.walk("explanations"): + for f in files: + file_path = os.path.join(subdir, f) + with open(file_path) as fh: + text = fh.read() + lowers = text.lower() + no_punctuation = lowers.translate(None, string.punctuation) + token_dict[f] = no_punctuation + + #this can take some time + tfidf = TfidfVectorizer(tokenizer=tokenize, stop_words='english') + tfs = tfidf.fit_transform(token_dict.values()) -- cgit v1.2.3-70-g09d2