From 4943ff41c71b80ed44075e26d1f84f1243b9831b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Baranowski?= Date: Tue, 2 Dec 2014 02:20:40 +0100 Subject: [PATCH 1/5] New filters: - language detection using NLTK - named entity extractor using ne_chunk method from NLTK - topic detection --- .../language_detection/language_detector.py | 25 ++++++ .../nlp/language_detection/language_filter.py | 50 +++++++++++ .../named_entity_extractor.py | 15 ++++ .../named_entity_filter.py | 51 +++++++++++ .../nlp/topic_detection/topic_detection.py | 51 +++++++++++ .../nlp/topic_detection/topic_filter.py | 84 +++++++++++++++++++ 6 files changed, 276 insertions(+) create mode 100644 examples/tweeter/nlp/language_detection/language_detector.py create mode 100644 examples/tweeter/nlp/language_detection/language_filter.py create mode 100644 examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py create mode 100644 examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py create mode 100644 examples/tweeter/nlp/topic_detection/topic_detection.py create mode 100644 examples/tweeter/nlp/topic_detection/topic_filter.py diff --git a/examples/tweeter/nlp/language_detection/language_detector.py b/examples/tweeter/nlp/language_detection/language_detector.py new file mode 100644 index 0000000..2df43ee --- /dev/null +++ b/examples/tweeter/nlp/language_detection/language_detector.py @@ -0,0 +1,25 @@ +from nltk import wordpunct_tokenize +from nltk.corpus import stopwords + +def detect_language(text): + + words = [word.lower() for word in wordpunct_tokenize(text)] + result = (None, -1) + + for language in stopwords.fileids(): + stopwords_set = set(stopwords.words(language)) + words_set = set(words) + common_elements = words_set.intersection(stopwords_set) + ratio = float(len(common_elements)) / len(stopwords_set) + + if ratio > result[1]: + result = (language, ratio) + + return result[0] + +if __name__=='__main__': + + text = "This is a test." + language = detect_language(text) + + print language diff --git a/examples/tweeter/nlp/language_detection/language_filter.py b/examples/tweeter/nlp/language_detection/language_filter.py new file mode 100644 index 0000000..51beb44 --- /dev/null +++ b/examples/tweeter/nlp/language_detection/language_filter.py @@ -0,0 +1,50 @@ +###START-CONF +##{ +##"object_name": "sentiment_analyses", +##"object_poi": "qpwo-2345", +##"auto-load": true, +##"remoting" : true, +##"parameters": [ +## { +## "name": "tweet", +## "description": "", +## "required": true, +## "type": "TweetString", +## "format": "", +## "state" : "ENGLISH" +## } +## ], +##"return": [ +## { +## "name": "tweet", +## "description": "topic detector", +## "required": true, +## "type": "TweetString", +## "format": "", +## "state" : "DANISH|DUTCH|ENGLISH|FINNISH|FRENCH|GERMAN|HUNGARIAN|ITALIAN|NORWEGIAN|PORTUGUESE|RUSSIAN|SPANISH|SWEDISH|TURKISH" +## } +## +## ] } +##END-CONF + +import re, os, time +import urllib2 +from random import randint +from pumpkin import PmkSeed +from language_detector import detect_language + +class topic_filter(PmkSeed.Seed): + + def __init__(self, context, poi=None): + PmkSeed.Seed.__init__(self, context,poi) + self.wd = self.context.getWorkingDir() + + def on_load(self): + print "Loading: " + self.__class__.__name__ + + def run(self, pkt, tweet): + m = re.search('W(\s+)(.*)(\n)', tweet, re.S) + if m: + tw = m.group(2) + language = detect_language(tw) + self.dispatch(pkt, tweet, str.upper(language)) diff --git a/examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py b/examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py new file mode 100644 index 0000000..4d91f7a --- /dev/null +++ b/examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py @@ -0,0 +1,15 @@ +from nltk import sent_tokenize, word_tokenize, pos_tag, ne_chunk + +def extract_named_entities(text): + sentences = sent_tokenize(text) + sentences = [word_tokenize(sent) for sent in sentences] + sentences = [pos_tag(sent) for sent in sentences] + result = [] + for sent in sentences: + result += [word[0] for word, tag in ne_chunk(sent, binary=True).pos() + if tag == 'NE'] + return result + +if __name__ == '__main__': + text = "This is test. Mr. Foobar is a bad person." + print extract_named_entities(text) diff --git a/examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py b/examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py new file mode 100644 index 0000000..a96fc80 --- /dev/null +++ b/examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py @@ -0,0 +1,51 @@ +###START-CONF +##{ +##"object_name": "sentiment_analyses", +##"object_poi": "qpwo-2345", +##"auto-load": true, +##"remoting" : true, +##"parameters": [ +## { +## "name": "tweet", +## "description": "", +## "required": true, +## "type": "TweetString", +## "format": "", +## "state" : "ENGLISH" +## } +## ], +##"return": [ +## { +## "name": "tweet", +## "description": "named entity extractor", +## "required": true, +## "type": "TweetString", +## "format": "", +## "state" : "ENTITIES" +## } +## +## ] } +##END-CONF + +import re, os, time +import urllib2 +from random import randint +from pumpkin import PmkSeed +from named_entity_extractor import extract_named_entities + +class topic_filter(PmkSeed.Seed): + + def __init__(self, context, poi=None): + PmkSeed.Seed.__init__(self, context,poi) + self.wd = self.context.getWorkingDir() + + def on_load(self): + print "Loading: " + self.__class__.__name__ + + def run(self, pkt, tweet): + m = re.search('W(\s+)(.*)(\n)', tweet, re.S) + if m: + tw = m.group(2) + entities = extract_named_entities(tw) + if len(entities) > 0: + self.dispatch(pkt, ",".join(entities), ENTITIES) diff --git a/examples/tweeter/nlp/topic_detection/topic_detection.py b/examples/tweeter/nlp/topic_detection/topic_detection.py new file mode 100644 index 0000000..92cdbce --- /dev/null +++ b/examples/tweeter/nlp/topic_detection/topic_detection.py @@ -0,0 +1,51 @@ +from nltk.corpus import reuters, movie_reviews +from operator import itemgetter +import nltk, pickle + + +class TopicDetector: + def __init__(self, path_to_data=None): + self._load_vector(path_to_data) + self.words = map(itemgetter(0), self.vector) + self.topics = ["movies"] + + def _load_vector(self, path_to_data): + if not path_to_data: + path_to_data = '/tmp/stats.pickle' + self.vector = None + try: + data_file = open(path_to_data, 'rb') + self.vector = pickle.load(data_file) + except IOError: + data_file = open(path_to_data, 'wb') + + all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words() if len(w) > 3) + all_words_r = nltk.FreqDist(w.lower() for w in reuters.words() if len(w) > 3) + + self.vector = [] + + for word in all_words.keys(): + ratio = 0 + try: + ratio = all_words.freq(word) / all_words_r.freq(word) + except ZeroDivisionError: + next + self.vector.append((word, ratio)) + self.vector.sort(key=itemgetter(1), reverse=True) + self.vector = self.vector[:200] + + pickle.dump(self.vector, data_file) + def is_topic(self, topic, text): + if topic not in self.topics: + None # todo: more topics than movies + + words = set([word.lower() for word in nltk.wordpunct_tokenize(text)]) + print self.words + inter = words.intersection(self.words) + print inter + return len(inter) > 1 + +if __name__ == '__main__': + td = TopicDetector() + print td.is_topic("movies", "What an amazing movie!") + print td.is_topic("movies", "Great premiere.") diff --git a/examples/tweeter/nlp/topic_detection/topic_filter.py b/examples/tweeter/nlp/topic_detection/topic_filter.py new file mode 100644 index 0000000..51f99d6 --- /dev/null +++ b/examples/tweeter/nlp/topic_detection/topic_filter.py @@ -0,0 +1,84 @@ +###START-CONF +##{ +##"object_name": "sentiment_analyses", +##"object_poi": "qpwo-2345", +##"auto-load": true, +##"remoting" : true, +##"parameters": [ +## { +## "name": "tweet", +## "description": "", +## "required": true, +## "type": "TweetString", +## "format": "", +## "state" : "ENGLISH" +## } +## ], +##"return": [ +## { +## "name": "tweet", +## "description": "topic detector", +## "required": true, +## "type": "TweetString", +## "format": "", +## "state" : "MOVIE" +## } +## +## ] } +##END-CONF + +import re, os, time +import urllib2 +from random import randint +from pumpkin import PmkSeed +from topic_detector import TopicDetector + +class topic_filter(PmkSeed.Seed): + + def __init__(self, context, poi=None): + PmkSeed.Seed.__init__(self, context,poi) + self.wd = self.context.getWorkingDir() + + def on_load(self): + print "Loading: " + self.__class__.__name__ + url = "URL-TO-DATA-FILE" + file_name = self.wd+"topic_detection_data.pickle" + self.get_net_file(url, file_name) + self.td = TopicDetector(file_name) + + def get_net_file(self, url, file_name): + #file_name = url.split('/')[-1] + downloaded = False + while not downloaded: + try: + u = urllib2.urlopen(url) + f = open(file_name, 'wb') + meta = u.info() + file_size = int(meta.getheaders("Content-Length")[0]) + self.logger.info ("Downloading: %s Bytes: %s" % (file_name, file_size)) + + file_size_dl = 0 + block_sz = 8192 + while True: + buffer = u.read(block_sz) + if not buffer: + break + + file_size_dl += len(buffer) + f.write(buffer) + #status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) + #status = status + chr(8)*(len(status)+1) + #print status, + f.close() + downloaded = True + except Exception as e: + self.logger.error("Error downloading, trying again....") + time.sleep(5) + pass + + def run(self, pkt, tweet): + m = re.search('W(\s+)(.*)(\n)', tweet, re.S) + if m: + tw = m.group(2) + if self.td.is_topic('movies', tw): + self.dispatch(pkt, tweet, "MOVIE") From 1977a78e502677b937417f3e1e7c8821b7ef86c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Baranowski?= Date: Wed, 3 Dec 2014 13:21:04 +0100 Subject: [PATCH 2/5] dependencies of seeds removed - content added to seed's files --- .../language_detection/language_detector.py | 25 --------- .../language_filter.py | 21 +++++++- .../named_entity_extractor.py | 15 ------ .../named_entity_filter.py | 14 ++++- .../nlp/topic_detection/topic_detection.py | 51 ------------------- .../nlp/{topic_detection => }/topic_filter.py | 47 ++++++++++++++++- 6 files changed, 77 insertions(+), 96 deletions(-) delete mode 100644 examples/tweeter/nlp/language_detection/language_detector.py rename examples/tweeter/nlp/{language_detection => }/language_filter.py (69%) delete mode 100644 examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py rename examples/tweeter/nlp/{named_entity_extractor => }/named_entity_filter.py (73%) delete mode 100644 examples/tweeter/nlp/topic_detection/topic_detection.py rename examples/tweeter/nlp/{topic_detection => }/topic_filter.py (62%) diff --git a/examples/tweeter/nlp/language_detection/language_detector.py b/examples/tweeter/nlp/language_detection/language_detector.py deleted file mode 100644 index 2df43ee..0000000 --- a/examples/tweeter/nlp/language_detection/language_detector.py +++ /dev/null @@ -1,25 +0,0 @@ -from nltk import wordpunct_tokenize -from nltk.corpus import stopwords - -def detect_language(text): - - words = [word.lower() for word in wordpunct_tokenize(text)] - result = (None, -1) - - for language in stopwords.fileids(): - stopwords_set = set(stopwords.words(language)) - words_set = set(words) - common_elements = words_set.intersection(stopwords_set) - ratio = float(len(common_elements)) / len(stopwords_set) - - if ratio > result[1]: - result = (language, ratio) - - return result[0] - -if __name__=='__main__': - - text = "This is a test." - language = detect_language(text) - - print language diff --git a/examples/tweeter/nlp/language_detection/language_filter.py b/examples/tweeter/nlp/language_filter.py similarity index 69% rename from examples/tweeter/nlp/language_detection/language_filter.py rename to examples/tweeter/nlp/language_filter.py index 51beb44..5715baa 100644 --- a/examples/tweeter/nlp/language_detection/language_filter.py +++ b/examples/tweeter/nlp/language_filter.py @@ -31,7 +31,8 @@ import urllib2 from random import randint from pumpkin import PmkSeed -from language_detector import detect_language +from nltk import wordpunct_tokenize +from nltk.corpus import stopwords class topic_filter(PmkSeed.Seed): @@ -42,9 +43,25 @@ def __init__(self, context, poi=None): def on_load(self): print "Loading: " + self.__class__.__name__ + def detect_language(text): + + words = [word.lower() for word in wordpunct_tokenize(text)] + result = (None, -1) + + for language in stopwords.fileids(): + stopwords_set = set(stopwords.words(language)) + words_set = set(words) + common_elements = words_set.intersection(stopwords_set) + ratio = float(len(common_elements)) / len(stopwords_set) + + if ratio > result[1]: + result = (language, ratio) + + return result[0] + def run(self, pkt, tweet): m = re.search('W(\s+)(.*)(\n)', tweet, re.S) if m: tw = m.group(2) - language = detect_language(tw) + language = self.detect_language(tw) self.dispatch(pkt, tweet, str.upper(language)) diff --git a/examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py b/examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py deleted file mode 100644 index 4d91f7a..0000000 --- a/examples/tweeter/nlp/named_entity_extractor/named_entity_extractor.py +++ /dev/null @@ -1,15 +0,0 @@ -from nltk import sent_tokenize, word_tokenize, pos_tag, ne_chunk - -def extract_named_entities(text): - sentences = sent_tokenize(text) - sentences = [word_tokenize(sent) for sent in sentences] - sentences = [pos_tag(sent) for sent in sentences] - result = [] - for sent in sentences: - result += [word[0] for word, tag in ne_chunk(sent, binary=True).pos() - if tag == 'NE'] - return result - -if __name__ == '__main__': - text = "This is test. Mr. Foobar is a bad person." - print extract_named_entities(text) diff --git a/examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py b/examples/tweeter/nlp/named_entity_filter.py similarity index 73% rename from examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py rename to examples/tweeter/nlp/named_entity_filter.py index a96fc80..6462169 100644 --- a/examples/tweeter/nlp/named_entity_extractor/named_entity_filter.py +++ b/examples/tweeter/nlp/named_entity_filter.py @@ -31,7 +31,7 @@ import urllib2 from random import randint from pumpkin import PmkSeed -from named_entity_extractor import extract_named_entities +from nltk import sent_tokenize, word_tokenize, pos_tag, ne_chunk class topic_filter(PmkSeed.Seed): @@ -42,10 +42,20 @@ def __init__(self, context, poi=None): def on_load(self): print "Loading: " + self.__class__.__name__ + def extract_named_entities(text): + sentences = sent_tokenize(text) + sentences = [word_tokenize(sent) for sent in sentences] + sentences = [pos_tag(sent) for sent in sentences] + result = [] + for sent in sentences: + result += [word[0] for word, tag in ne_chunk(sent, binary=True).pos() + if tag == 'NE'] + return result + def run(self, pkt, tweet): m = re.search('W(\s+)(.*)(\n)', tweet, re.S) if m: tw = m.group(2) - entities = extract_named_entities(tw) + entities = self.extract_named_entities(tw) if len(entities) > 0: self.dispatch(pkt, ",".join(entities), ENTITIES) diff --git a/examples/tweeter/nlp/topic_detection/topic_detection.py b/examples/tweeter/nlp/topic_detection/topic_detection.py deleted file mode 100644 index 92cdbce..0000000 --- a/examples/tweeter/nlp/topic_detection/topic_detection.py +++ /dev/null @@ -1,51 +0,0 @@ -from nltk.corpus import reuters, movie_reviews -from operator import itemgetter -import nltk, pickle - - -class TopicDetector: - def __init__(self, path_to_data=None): - self._load_vector(path_to_data) - self.words = map(itemgetter(0), self.vector) - self.topics = ["movies"] - - def _load_vector(self, path_to_data): - if not path_to_data: - path_to_data = '/tmp/stats.pickle' - self.vector = None - try: - data_file = open(path_to_data, 'rb') - self.vector = pickle.load(data_file) - except IOError: - data_file = open(path_to_data, 'wb') - - all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words() if len(w) > 3) - all_words_r = nltk.FreqDist(w.lower() for w in reuters.words() if len(w) > 3) - - self.vector = [] - - for word in all_words.keys(): - ratio = 0 - try: - ratio = all_words.freq(word) / all_words_r.freq(word) - except ZeroDivisionError: - next - self.vector.append((word, ratio)) - self.vector.sort(key=itemgetter(1), reverse=True) - self.vector = self.vector[:200] - - pickle.dump(self.vector, data_file) - def is_topic(self, topic, text): - if topic not in self.topics: - None # todo: more topics than movies - - words = set([word.lower() for word in nltk.wordpunct_tokenize(text)]) - print self.words - inter = words.intersection(self.words) - print inter - return len(inter) > 1 - -if __name__ == '__main__': - td = TopicDetector() - print td.is_topic("movies", "What an amazing movie!") - print td.is_topic("movies", "Great premiere.") diff --git a/examples/tweeter/nlp/topic_detection/topic_filter.py b/examples/tweeter/nlp/topic_filter.py similarity index 62% rename from examples/tweeter/nlp/topic_detection/topic_filter.py rename to examples/tweeter/nlp/topic_filter.py index 51f99d6..49795cc 100644 --- a/examples/tweeter/nlp/topic_detection/topic_filter.py +++ b/examples/tweeter/nlp/topic_filter.py @@ -31,7 +31,9 @@ import urllib2 from random import randint from pumpkin import PmkSeed -from topic_detector import TopicDetector +from nltk.corpus import reuters, movie_reviews +from operator import itemgetter +import nltk, pickle class topic_filter(PmkSeed.Seed): @@ -82,3 +84,46 @@ def run(self, pkt, tweet): tw = m.group(2) if self.td.is_topic('movies', tw): self.dispatch(pkt, tweet, "MOVIE") + + +class TopicDetector: + def __init__(self, path_to_data=None): + self._load_vector(path_to_data) + self.words = map(itemgetter(0), self.vector) + self.topics = ["movies"] + + def _load_vector(self, path_to_data): + if not path_to_data: + path_to_data = '/tmp/stats.pickle' + self.vector = None + try: + data_file = open(path_to_data, 'rb') + self.vector = pickle.load(data_file) + except IOError: + data_file = open(path_to_data, 'wb') + + all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words() if len(w) > 3) + all_words_r = nltk.FreqDist(w.lower() for w in reuters.words() if len(w) > 3) + + self.vector = [] + + for word in all_words.keys(): + ratio = 0 + try: + ratio = all_words.freq(word) / all_words_r.freq(word) + except ZeroDivisionError: + next + self.vector.append((word, ratio)) + self.vector.sort(key=itemgetter(1), reverse=True) + self.vector = self.vector[:200] + + pickle.dump(self.vector, data_file) + def is_topic(self, topic, text): + if topic not in self.topics: + None # todo: more topics than movies + + words = set([word.lower() for word in nltk.wordpunct_tokenize(text)]) + print self.words + inter = words.intersection(self.words) + print inter + return len(inter) > 1 From 3934ca5ab40bcefd0387b5338b361c88a532af39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Baranowski?= Date: Wed, 3 Dec 2014 21:00:52 +0100 Subject: [PATCH 3/5] fixed after extensive testing --- examples/tweeter/nlp/language_filter.py | 26 +++++++++++++-------- examples/tweeter/nlp/named_entity_filter.py | 24 +++++++++++-------- examples/tweeter/nlp/topic_filter.py | 19 ++++++++------- 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/examples/tweeter/nlp/language_filter.py b/examples/tweeter/nlp/language_filter.py index 5715baa..9e96d84 100644 --- a/examples/tweeter/nlp/language_filter.py +++ b/examples/tweeter/nlp/language_filter.py @@ -1,6 +1,6 @@ ###START-CONF ##{ -##"object_name": "sentiment_analyses", +##"object_name": "language_filter", ##"object_poi": "qpwo-2345", ##"auto-load": true, ##"remoting" : true, @@ -11,7 +11,7 @@ ## "required": true, ## "type": "TweetString", ## "format": "", -## "state" : "ENGLISH" +## "state" : "RAW" ## } ## ], ##"return": [ @@ -21,12 +21,14 @@ ## "required": true, ## "type": "TweetString", ## "format": "", -## "state" : "DANISH|DUTCH|ENGLISH|FINNISH|FRENCH|GERMAN|HUNGARIAN|ITALIAN|NORWEGIAN|PORTUGUESE|RUSSIAN|SPANISH|SWEDISH|TURKISH" +## "state" : "ENGLISH" ## } ## ## ] } ##END-CONF +# supports: DANISH|DUTCH|ENGLISH|FINNISH|FRENCH|GERMAN|HUNGARIAN|ITALIAN|NORWEGIAN|PORTUGUESE|RUSSIAN|SPANISH|SWEDISH|TURKISH + import re, os, time import urllib2 from random import randint @@ -34,7 +36,7 @@ from nltk import wordpunct_tokenize from nltk.corpus import stopwords -class topic_filter(PmkSeed.Seed): +class language_filter(PmkSeed.Seed): def __init__(self, context, poi=None): PmkSeed.Seed.__init__(self, context,poi) @@ -43,7 +45,7 @@ def __init__(self, context, poi=None): def on_load(self): print "Loading: " + self.__class__.__name__ - def detect_language(text): + def detect_language(self, text): words = [word.lower() for word in wordpunct_tokenize(text)] result = (None, -1) @@ -60,8 +62,12 @@ def detect_language(text): return result[0] def run(self, pkt, tweet): - m = re.search('W(\s+)(.*)(\n)', tweet, re.S) - if m: - tw = m.group(2) - language = self.detect_language(tw) - self.dispatch(pkt, tweet, str.upper(language)) + for t in tweet: + m = re.search('W(\s+)(.*)(\n)', t, re.S) + if m: + tw = m.group(2) + + if len(tw) > 10 and type(tw) == str: + language = self.detect_language(tw) + if language == 'english': + self.dispatch(pkt, t, 'ENGLISH') diff --git a/examples/tweeter/nlp/named_entity_filter.py b/examples/tweeter/nlp/named_entity_filter.py index 6462169..b256ab4 100644 --- a/examples/tweeter/nlp/named_entity_filter.py +++ b/examples/tweeter/nlp/named_entity_filter.py @@ -1,6 +1,6 @@ ###START-CONF ##{ -##"object_name": "sentiment_analyses", +##"object_name": "named_entity_filter", ##"object_poi": "qpwo-2345", ##"auto-load": true, ##"remoting" : true, @@ -11,7 +11,7 @@ ## "required": true, ## "type": "TweetString", ## "format": "", -## "state" : "ENGLISH" +## "state" : "MOVIE" ## } ## ], ##"return": [ @@ -33,7 +33,7 @@ from pumpkin import PmkSeed from nltk import sent_tokenize, word_tokenize, pos_tag, ne_chunk -class topic_filter(PmkSeed.Seed): +class named_entity_filter(PmkSeed.Seed): def __init__(self, context, poi=None): PmkSeed.Seed.__init__(self, context,poi) @@ -42,7 +42,7 @@ def __init__(self, context, poi=None): def on_load(self): print "Loading: " + self.__class__.__name__ - def extract_named_entities(text): + def extract_named_entities(self, text): sentences = sent_tokenize(text) sentences = [word_tokenize(sent) for sent in sentences] sentences = [pos_tag(sent) for sent in sentences] @@ -53,9 +53,13 @@ def extract_named_entities(text): return result def run(self, pkt, tweet): - m = re.search('W(\s+)(.*)(\n)', tweet, re.S) - if m: - tw = m.group(2) - entities = self.extract_named_entities(tw) - if len(entities) > 0: - self.dispatch(pkt, ",".join(entities), ENTITIES) + for t in tweet: + m = re.search('W(\s+)(.*)(\n)', t, re.S) + if m: + tw = m.group(2) + self.logger.info("named_entity_filter: " + tw) + entities = self.extract_named_entities(tw) + if len(entities) > 0: + self.logger.info("named_entity_filter: |" + "| ".join(entities)) + self.dispatch(pkt, ", ".join(entities), 'ENTITIES') + diff --git a/examples/tweeter/nlp/topic_filter.py b/examples/tweeter/nlp/topic_filter.py index 49795cc..2ff86c6 100644 --- a/examples/tweeter/nlp/topic_filter.py +++ b/examples/tweeter/nlp/topic_filter.py @@ -1,6 +1,6 @@ ###START-CONF ##{ -##"object_name": "sentiment_analyses", +##"object_name": "topic_filter", ##"object_poi": "qpwo-2345", ##"auto-load": true, ##"remoting" : true, @@ -43,7 +43,7 @@ def __init__(self, context, poi=None): def on_load(self): print "Loading: " + self.__class__.__name__ - url = "URL-TO-DATA-FILE" + url = "https://www.dropbox.com/s/qn6o8r3liq5jxv4/topic_detection_data.pickle?dl=1" file_name = self.wd+"topic_detection_data.pickle" self.get_net_file(url, file_name) self.td = TopicDetector(file_name) @@ -79,11 +79,13 @@ def get_net_file(self, url, file_name): pass def run(self, pkt, tweet): - m = re.search('W(\s+)(.*)(\n)', tweet, re.S) - if m: - tw = m.group(2) - if self.td.is_topic('movies', tw): - self.dispatch(pkt, tweet, "MOVIE") + for t in tweet: + m = re.search('W(\s+)(.*)(\n)', t, re.S) + if m: + tw = m.group(2) + if self.td.is_topic('movies', tw): + self.logger.info("topic_filter: topic found in " + tw) + self.dispatch(pkt, t, "MOVIE") class TopicDetector: @@ -123,7 +125,6 @@ def is_topic(self, topic, text): None # todo: more topics than movies words = set([word.lower() for word in nltk.wordpunct_tokenize(text)]) - print self.words inter = words.intersection(self.words) - print inter return len(inter) > 1 + From c958ee4812c42abffad294533de27c44e165dd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Baranowski?= Date: Fri, 5 Dec 2014 12:16:52 +0100 Subject: [PATCH 4/5] consumes one tweet per message (instead of a list) to use with option "--gonzales" --- examples/tweeter/nlp/language_filter.py | 16 +++++++--------- examples/tweeter/nlp/named_entity_filter.py | 17 ++++++++--------- examples/tweeter/nlp/topic_filter.py | 13 ++++++------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/examples/tweeter/nlp/language_filter.py b/examples/tweeter/nlp/language_filter.py index 9e96d84..455c9a9 100644 --- a/examples/tweeter/nlp/language_filter.py +++ b/examples/tweeter/nlp/language_filter.py @@ -62,12 +62,10 @@ def detect_language(self, text): return result[0] def run(self, pkt, tweet): - for t in tweet: - m = re.search('W(\s+)(.*)(\n)', t, re.S) - if m: - tw = m.group(2) - - if len(tw) > 10 and type(tw) == str: - language = self.detect_language(tw) - if language == 'english': - self.dispatch(pkt, t, 'ENGLISH') + m = re.search('W(\s+)(.*)(\n)', tweet, re.S) + if m: + tw = m.group(2) + if len(tw) > 10: + language = self.detect_language(tw) + if language == 'english': + self.dispatch(pkt, tweet, 'ENGLISH') diff --git a/examples/tweeter/nlp/named_entity_filter.py b/examples/tweeter/nlp/named_entity_filter.py index b256ab4..353c43c 100644 --- a/examples/tweeter/nlp/named_entity_filter.py +++ b/examples/tweeter/nlp/named_entity_filter.py @@ -53,13 +53,12 @@ def extract_named_entities(self, text): return result def run(self, pkt, tweet): - for t in tweet: - m = re.search('W(\s+)(.*)(\n)', t, re.S) - if m: - tw = m.group(2) - self.logger.info("named_entity_filter: " + tw) - entities = self.extract_named_entities(tw) - if len(entities) > 0: - self.logger.info("named_entity_filter: |" + "| ".join(entities)) - self.dispatch(pkt, ", ".join(entities), 'ENTITIES') + m = re.search('W(\s+)(.*)(\n)', tweet, re.S) + if m: + tw = m.group(2) + self.logger.info("named_entity_filter: " + tw) + entities = self.extract_named_entities(tw) + if len(entities) > 0: + self.logger.info("named_entity_filter: |" + "| ".join(entities)) + self.dispatch(pkt, ", ".join(entities), 'ENTITIES') diff --git a/examples/tweeter/nlp/topic_filter.py b/examples/tweeter/nlp/topic_filter.py index 2ff86c6..b6941f9 100644 --- a/examples/tweeter/nlp/topic_filter.py +++ b/examples/tweeter/nlp/topic_filter.py @@ -79,13 +79,12 @@ def get_net_file(self, url, file_name): pass def run(self, pkt, tweet): - for t in tweet: - m = re.search('W(\s+)(.*)(\n)', t, re.S) - if m: - tw = m.group(2) - if self.td.is_topic('movies', tw): - self.logger.info("topic_filter: topic found in " + tw) - self.dispatch(pkt, t, "MOVIE") + m = re.search('W(\s+)(.*)(\n)', tweet, re.S) + if m: + tw = m.group(2) + if self.td.is_topic('movies', tw): + self.logger.info("topic_filter: topic found in " + tw) + self.dispatch(pkt, tweet, "MOVIE") class TopicDetector: From 57d65e1475174c114a7087eb82d22af22b888dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Baranowski?= Date: Fri, 5 Dec 2014 12:17:45 +0100 Subject: [PATCH 5/5] sentiment analyses included in nlp workflow --- examples/tweeter/nlp/named_entity_filter.py | 2 +- examples/tweeter/soa/sentiment_analyses.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/tweeter/nlp/named_entity_filter.py b/examples/tweeter/nlp/named_entity_filter.py index 353c43c..6a08296 100644 --- a/examples/tweeter/nlp/named_entity_filter.py +++ b/examples/tweeter/nlp/named_entity_filter.py @@ -11,7 +11,7 @@ ## "required": true, ## "type": "TweetString", ## "format": "", -## "state" : "MOVIE" +## "state" : "POSITIVE" ## } ## ], ##"return": [ diff --git a/examples/tweeter/soa/sentiment_analyses.py b/examples/tweeter/soa/sentiment_analyses.py index 9ceb311..9a3a312 100644 --- a/examples/tweeter/soa/sentiment_analyses.py +++ b/examples/tweeter/soa/sentiment_analyses.py @@ -11,7 +11,7 @@ ## "required": true, ## "type": "TweetString", ## "format": "", -## "state" : "ENGLISH" +## "state" : "MOVIE" ## } ## ], ##"return": [ @@ -21,7 +21,7 @@ ## "required": true, ## "type": "TweetString", ## "format": "", -## "state" : "POSITIVE|NEGATIVE" +## "state" : "POSITIVE" ## } ## ## ] } @@ -100,5 +100,5 @@ def run(self, pkt, tweet): tw = m.group(2) if self.check(tw): self.dispatch(pkt, tweet, "POSITIVE") - else: - self.dispatch(pkt, tweet, "NEGATIVE") + # else: + # self.dispatch(pkt, tweet, "NEGATIVE")