Reverse Pull Request

.gitignore

@@ -23,6 +23,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+.DS_Store
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -96,4 +97,4 @@ pip-selfcheck.json
 share
 
 #misc
-*.sqlite
+*.sqlite

Procfile

@@ -0,0 +1 @@
+web: ./run

README.md

@@ -0,0 +1,33 @@
+# PYSEARCH
+
+**Authors:** Marc Fieser, Sera Smith, and Ben Shields
+
+**Live URL:** https://vascodagama.herokuapp.com/
+
+## Getting Started:
+```
+- cd <directory containing this file>
+
+- $VENV/bin/pip install -e .
+
+- $VENV/bin/initialize_db development.ini
+
+- $VENV/bin/pserve development.ini
+```
+
+## Global Variables:
+
+harvester.py: 
+NUM_OF_OCCURANCES -- number of times a word must appear in order for harvester to add word to database keyword table.
+
+crawler.py:
+CRAWL_COUNT -- crawl page count parameter
+DEPTH_LEVEL -- depth per page crawled
+
+pipelines.py:
+MINIMUM_MATCHES -- minimum number of word matches for url to be sent to database matches table.
+
+## License:
+MIT
+
+**Copyright 2017**

development.ini

@@ -14,7 +14,8 @@ pyramid.default_locale_name = en
 pyramid.includes =
     pyramid_debugtoolbar
 
-sqlalchemy.url = sqlite:///%(here)s/pysearch.sqlite
+sqlalchemy.url = postgres://@localhost:5432/pysearch
+
 
 # By default, the toolbar only appears for clients from IP addresses
 # '127.0.0.1' and '::1'.

pysearch/__init__.py

@@ -6,7 +6,7 @@
 def main(global_config, **settings):
     """ This function returns a Pyramid WSGI application.
     """
-    settings["sqlachemy.url"] = os.environ["DATABASE_URL"]
+    # settings["sqlachemy.url"] = os.environ["DATABASE_URL"]
     config = Configurator(settings=settings)
     config.include('pyramid_jinja2')
     config.include('.models')

pysearch/harvester/middlewares.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class HarvesterSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

pysearch/harvester/pipelines.py

@@ -0,0 +1,80 @@
+"""pipelines.py."""
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+from sqlalchemy.orm import sessionmaker
+from pysearch.models import Keyword, Match
+
+
+from pysearch.models.meta import Base
+from sqlalchemy.engine.url import URL
+from sqlalchemy import create_engine
+
+DATABASE = {
+    'drivername': 'postgres',
+    'host': 'localhost',
+    'port': '5432',
+    'database': 'pysearch'
+}
+
+MINIMUM_MATCHES = 5
+
+
+def db_connect():
+    """Perform database connection using database settings from settings.py. Returns sqlalchemy engine instance."""
+    return create_engine(URL(**DATABASE))
+
+
+def create_keyword_table(engine):
+    """Create tables."""
+    Base.metadata.create_all(engine)
+
+
+class CrawlerPipeline(object):
+    """Crawler pipeline for comparing scraped items with items in the database."""
+
+    def __init__(self):
+        """Initialize database connection and sessionmaker. Creates matches table."""
+        engine = db_connect()
+        create_keyword_table(engine)
+        self.Session = sessionmaker(bind=engine)
+
+    def process_item(self, item, spider):
+        """Save matches in the database.
+
+        This method is called for every item pipeline component.
+
+        """
+        if spider.name is 'crawler':
+            session = self.Session()
+            try:
+                db_words = session.query(Keyword).all()
+                match_words = []
+                for word in db_words:
+                    if word.keyword in item['words']:
+                        match = {
+                            'word': word.keyword,
+                            'key_weight': word.keyword_weight,
+                            'count': item['words'][word.keyword],
+                            'url': item['url']
+                        }
+                        match_words.append(match)
+                if len(match_words) > MINIMUM_MATCHES:
+                    to_add = []
+                    for match in match_words:
+                        new_keyword = Match(keyword=match['word'], keyword_weight=match['key_weight'], page_url=match['url'], count=match['count'])
+                        to_add.append(new_keyword)
+                        print('Pysearch Database Updated...')
+                    session.add_all(to_add)
+                    session.commit()
+            except:
+                session.rollback()
+                raise
+            finally:
+                session.close()
+
+        return item

pysearch/harvester/settings.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for harvester project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     http://doc.scrapy.org/en/latest/topics/settings.html
+#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'harvester'
+
+SPIDER_MODULES = ['harvester.spiders']
+NEWSPIDER_MODULE = 'harvester.spiders'
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'harvester (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'harvester.middlewares.HarvesterSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'harvester.middlewares.MyCustomDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
+ITEM_PIPELINES = {
+    'harvester.pipelines.HarvesterPipeline': 300,
+    # 'harvester.pipelines.CrawlerPipeline': 300,
+}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

pysearch/harvester/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

pysearch/harvester/spiders/crawler.py

@@ -0,0 +1,80 @@
+"""Spider for crawling."""
+import collections
+from stop_words import get_stop_words
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import Rule, CrawlSpider
+from scrapy.item import Item, Field
+
+CRAWL_COUNT = 10
+DEPTH_LEVEL = 10
+
+
+class MyItem(Item):
+    """Item container for scraping."""
+
+    url = Field()
+    words = Field()
+
+
+class CrawlingSpider(CrawlSpider):
+    """Spider for harvesting words from a URL."""
+
+    name = "crawler"
+    custom_settings = {
+        'ITEM_PIPELINES': {
+            'pysearch.harvester.pipelines.CrawlerPipeline': 300,
+        }
+    }
+
+    rules = (Rule(LinkExtractor(allow=("", ),), callback="parse_items", follow=True),)
+
+    def __init__(self, url=None, *args, **kwargs):
+        """Initialize a harvest spider."""
+        super(CrawlingSpider, self).__init__(*args, **kwargs)
+
+    def parse_items(self, response):
+        """Get links from site."""
+        item = MyItem()
+        words = []
+        stop_words = get_stop_words('english')
+        p = response.css('p::text').extract()
+        for each in p:
+            words.extend(each.split())
+        words = lower_list(words)
+        word_count = collections.Counter(words)
+        for key in list(word_count.keys()):
+            if key in stop_words:
+                del word_count[key]
+        item['words'] = word_count
+        item['url'] = response.url
+        yield item
+
+
+def crawl(url):
+    """Initialize crawling sequence."""
+    settings = get_project_settings()
+    settings.url = url
+    settings["CLOSESPIDER_PAGECOUNT"] = CRAWL_COUNT
+    settings["DEPTH_LEVEL"] = DEPTH_LEVEL
+    process = CrawlerProcess(settings)
+
+    class ThisSpider(CrawlingSpider):
+        """Create a spider to crawl with."""
+
+        start_urls = [url]
+    process.crawl(ThisSpider)
+    process.start()
+
+
+def lower_list(list_in):
+    """Return a list with all words lowercase."""
+    list_out = []
+    for each in list_in:
+        list_out.append(each.lower())
+    return list_out
+
+if __name__ == '__main__':
+    import sys
+    crawl(sys.argv[1])