Update to lucene 5.2.1

.gitignore

@@ -5,3 +5,4 @@ clucy*.jar
 pom.xml
 pom.xml.asc
 .lein-failures
+.nrepl-port

.travis.yml

@@ -3,6 +3,9 @@ language: clojure
 lein: lein2
 
 jdk:
-  - openjdk6
   - openjdk7
-  - oraclejdk7
+  - oraclejdk7
+  - oraclejdk8
+
+after_script:
+  - bash -ex test/coveralls.sh

README.md

@@ -1,7 +1,11 @@
 Clucy
 =====
 
-[![Build Status](https://secure.travis-ci.org/weavejester/clucy.png?branch=master)](http://travis-ci.org/weavejester/clucy)
+[![License EPL](https://img.shields.io/badge/license-EPL-yellow.svg)](https://www.eclipse.org/legal/epl-v10.html)
+[![Build Status](https://travis-ci.org/kostafey/clucy.svg?branch=master)](https://travis-ci.org/kostafey/clucy)
+[![Clojars Project](https://img.shields.io/badge/clojars-clucy-blue.svg)](https://clojars.org/org.clojars.kostafey/clucy)
+[![Coverage Status](https://coveralls.io/repos/kostafey/clucy/badge.svg?branch=master)](https://coveralls.io/github/kostafey/clucy?branch=master)
+[![Dependencies Status](https://jarkeeper.com/kostafey/clucy/status.svg)](https://jarkeeper.com/kostafey/clucy)
 
 Clucy is a Clojure interface to [Lucene](http://lucene.apache.org/).
 
@@ -11,11 +15,13 @@ Installation
 To install Clucy, add the following dependency to your `project.clj`
 file:
 
-    [clucy "0.4.0"]
+[![Clojars Project](http://clojars.org/org.clojars.kostafey/clucy/latest-version.svg)](http://clojars.org/org.clojars.kostafey/clucy)
 
 Usage
 -----
 
+#### Search in documents
+
 To use Clucy, first require it:
 
     (ns example
@@ -51,6 +57,59 @@ scientists...
 
     (clucy/search-and-delete index "job:scientist")
 
+#### Search text positions in single document
+
+```clojure
+(ns example
+  (:use [clucy.core
+         clucy.analyzers
+         clucy.positions-searcher]))
+
+(binding [*analyzer* (make-analyzer :class :en)]
+  (let [test-text "This is the house that Jack built.
+                   This is the malt
+                   That lay in the house that Jack built."
+        index (doto (memory-index)
+                (add (set-field-params
+                      test-text
+                      {:positions-offsets true
+                       :vector-positions true})))
+        searcher (make-dict-searcher
+                  #{"house"
+                    "lay"
+                    "Jack built"})
+        result-iter (searcher index)]
+    (sort-by second
+             (show-text-matches result-iter test-text))))
+```
+
+    => (["house" 12] ["Jack built" 23] ["lay" 95] ["house" 106] ["Jack built" 117])
+
+#### Statistics for single document
+
+```clojure
+(ns example
+  (:use clucy.core
+        clucy.analyzers
+        clucy.document-statistics))
+
+(binding [*analyzer* (make-analyzer :class :en)]
+  (let [index (doto (memory-index)
+                (add (set-field-params
+                      "This is the house that Jack built.
+                       This is the malt
+                       That lay in the house that Jack built."
+                      {:positions-offsets true})))
+        iterator (get-top-words-iterator index 2)]
+    {:word-count (get-word-count index)
+     :most-frequent (iterator)}))
+
+    => {:word-count 8,
+        :most-frequent (["built" {:count 2, :pos ([130 135] [28 33])}]
+                        ["hous" {:count 2, :pos ([114 119] [12 17])}]
+                        ["jack" {:count 2, :pos ([125 129] [23 27])}])}
+```
+
 Storing Fields
 --------------
 

project.clj

@@ -1,15 +1,15 @@
-(defproject clucy "0.4.0"
+(defproject org.clojars.kostafey/clucy "0.5.5.0"
   :description "A Clojure interface to the Lucene search engine"
-  :url "http://github/weavejester/clucy"
-  :dependencies [[org.clojure/clojure "1.4.0"]
-                 [org.apache.lucene/lucene-core "4.2.0"]
-                 [org.apache.lucene/lucene-queryparser "4.2.0"]
-                 [org.apache.lucene/lucene-analyzers-common "4.2.0"]
-                 [org.apache.lucene/lucene-highlighter "4.2.0"]]
+  :url "http://github/kostafey/clucy"
+  :dependencies [[org.clojure/clojure "1.8.0"]
+                 [org.apache.lucene/lucene-core "5.5.0"]
+                 [org.apache.lucene/lucene-queryparser "5.5.0"]
+                 [org.apache.lucene/lucene-analyzers-common "5.5.0"]
+                 [org.apache.lucene/lucene-highlighter "5.5.0"]
+                 [me.raynes/fs "1.4.6"]]
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
-  :profiles {:1.4  {:dependencies [[org.clojure/clojure "1.4.0"]]}
-             :1.5  {:dependencies [[org.clojure/clojure "1.5.0"]]}
-             :1.6  {:dependencies [[org.clojure/clojure "1.6.0-master-SNAPSHOT"]]}}
-  :codox {:src-dir-uri "http://github/weavejester/clucy/blob/master"
-          :src-linenum-anchor-prefix "L"})
+  :profiles {:1.6  {:dependencies [[org.clojure/clojure "1.6.0"]]}
+             :1.7  {:dependencies [[org.clojure/clojure "1.7.0"]]}
+             :1.8  {:dependencies [[org.clojure/clojure "1.8.0"]]}}
+  :plugins [[lein-cloverage "1.0.6"]])

src/clucy/analyzers.clj

@@ -0,0 +1,198 @@
+(ns clucy.analyzers
+  (:use clucy.util)
+  (:require [clojure.java.io :as io])
+  (:import
+   (java.io InputStream)
+   (java.nio.charset StandardCharsets)
+   (org.apache.lucene.analysis.util WordlistLoader
+                                    CharArraySet)
+   (org.apache.lucene.util IOUtils)
+   (org.apache.lucene.analysis.Analyzer$TokenStreamComponents)
+   (org.apache.lucene.analysis Analyzer
+                               TokenStream
+                               Tokenizer
+                               TokenFilter
+                               CachingTokenFilter)
+   (org.apache.lucene.analysis.standard StandardAnalyzer
+                                        ClassicAnalyzer
+                                        StandardFilter
+                                        StandardTokenizer
+                                        ClassicTokenizer
+                                        ClassicFilter)
+   (org.apache.lucene.analysis.snowball SnowballFilter)
+   (org.apache.lucene.analysis.ar ArabicAnalyzer)
+   (org.apache.lucene.analysis.bg BulgarianAnalyzer)
+   (org.apache.lucene.analysis.de GermanAnalyzer
+                                  GermanLightStemFilter)
+   (org.apache.lucene.analysis.en EnglishAnalyzer
+                                  EnglishMinimalStemFilter)
+   (org.apache.lucene.analysis.fr FrenchAnalyzer
+                                  FrenchLightStemFilter)
+   (org.apache.lucene.analysis.ru RussianAnalyzer
+                                  RussianLightStemFilter)
+   (org.apache.lucene.analysis.core LowerCaseFilter
+                                    StopFilter
+                                    WhitespaceTokenizer
+                                    LetterTokenizer
+                                    KeywordTokenizer
+                                    LowerCaseTokenizer)
+   (org.apache.lucene.analysis.path PathHierarchyTokenizer)
+   (org.apache.lucene.analysis.wikipedia WikipediaTokenizer)
+   (org.apache.lucene.analysis.miscellaneous SetKeywordMarkerFilter
+                                             LengthFilter)
+   (org.tartarus.snowball.ext EnglishStemmer
+                              FrenchStemmer
+                              GermanStemmer
+                              RussianStemmer)))
+
+(def analysers-class-map
+  {:basic    Analyzer
+   :standard StandardAnalyzer
+   :classic  ClassicAnalyzer
+   :ar       ArabicAnalyzer
+   :bg       BulgarianAnalyzer
+   :fr       FrenchAnalyzer
+   :de       GermanAnalyzer
+   :en       EnglishAnalyzer
+   :ru       RussianAnalyzer})
+
+(def tokenizers-class-map
+  {:standard       StandardTokenizer
+   :whitespace     WhitespaceTokenizer
+   :letter         LetterTokenizer
+   :classic        ClassicTokenizer
+   :keyword        KeywordTokenizer
+   :lowercase      LowerCaseTokenizer
+   :path-hierarchy PathHierarchyTokenizer
+   :wikipedia      WikipediaTokenizer})
+
+(def filters-class-map
+  {:standard      StandardFilter
+   :snowball      SnowballFilter
+   :classic       ClassicFilter
+   :caching-token CachingTokenFilter})
+
+(def stemmers-class-map
+  {:en EnglishStemmer
+   :fr FrenchStemmer
+   :de GermanStemmer
+   :ru RussianStemmer
+   :en-min EnglishMinimalStemFilter
+   :fr-light FrenchLightStemFilter
+   :de-light GermanLightStemFilter
+   :ru-light RussianLightStemFilter})
+
+(defn- build-analyzer
+  ([analyzer-class]
+   (.newInstance (analysers-class-map analyzer-class)))
+  ([analyzer-class stop-words]
+   (let [ctor (.getConstructor (analysers-class-map analyzer-class)
+                               (into-array [CharArraySet]))]
+     (.newInstance ctor (into-array [stop-words]))))
+  ([analyzer-class stop-words stem-exclusion-words]
+   (let [ctor (.getConstructor (analysers-class-map analyzer-class)
+                               (into-array [CharArraySet
+                                            CharArraySet]))]
+     (.newInstance ctor (into-array [stop-words
+                                     stem-exclusion-words])))))
+
+(defn- get-analyzer [analyzer-class stop-words stem-exclusion-words]
+  (assert (not (and (some #{analyzer-class} [:standard :classic])
+                    (not (nil? stem-exclusion-words))))
+          "Can't set stem-exclusion-words for Standard or Classic Analyzer.")
+  (cond
+    (and stop-words stem-exclusion-words) (build-analyzer
+                                           analyzer-class
+                                           stop-words
+                                           stem-exclusion-words)
+    (boolean stop-words) (build-analyzer analyzer-class stop-words)
+    :else (build-analyzer analyzer-class)))
+
+
+(defn- get-tokenizer [key-or-object]
+  (if (instance? Tokenizer key-or-object)
+    key-or-object
+    (.newInstance (tokenizers-class-map key-or-object))))
+
+(defn make-analyzer
+  ([] (make-analyzer :class :standard))
+  ([& {:keys [class
+              version
+              stop-words
+              stem-exclusion-words
+              tokenizer
+              filter
+              stemmer
+              lower-case
+              length-filter]
+       :or {class :basic
+            version org.apache.lucene.util.Version/LATEST
+            stop-words nil
+            stem-exclusion-words nil
+            tokenizer :standard
+            filter :standard
+            stemmer nil
+            lower-case true
+            length-filter nil}}]
+   (let [analyzer
+         (if (not (= :basic class))
+           ;; ------------------------------------------------------------
+           ;; Use pre-defined analyzer class.
+           ;; All params except stop-words, stem-exclusion-words and version
+           ;; are ignored.
+           (get-analyzer class stop-words stem-exclusion-words)
+           ;; ------------------------------------------------------------
+           ;; Custom analyser.
+           (proxy [Analyzer] []
+             (createComponents [fieldName]
+               (let [^Tokenizer source (get-tokenizer tokenizer)
+                     ^TokenStream result (.newInstance
+                                          (.getConstructor
+                                           (filters-class-map filter)
+                                           (into-array [TokenStream]))
+                                          (into-array [source]))
+                     result (if lower-case (LowerCaseFilter. result) result)
+                     result (if length-filter (LengthFilter.
+                                               result
+                                               (first length-filter)
+                                               (second length-filter)) result)
+                     result (if stop-words (StopFilter. result stop-words) result)
+                     result (if stem-exclusion-words
+                              (SetKeywordMarkerFilter. result stem-exclusion-words)
+                              result)
+                     result (if stemmer
+                              ;; for light stemmers
+                              (if (or (ends-with (name stemmer) "light")
+                                      (ends-with (name stemmer) "min"))
+                                (.newInstance
+                                 (.getConstructor
+                                  (stemmers-class-map stemmer)
+                                  (into-array [TokenStream]))
+                                 (into-array [result]))
+                                ;; for snowball stemmers
+                                (SnowballFilter.
+                                 result
+                                 (.newInstance (stemmers-class-map stemmer))))
+                              result)]
+                 (org.apache.lucene.analysis.Analyzer$TokenStreamComponents.
+                  source result)))))]
+     (.setVersion analyzer version)
+     analyzer)))
+
+(defn file->wordset ^CharArraySet [^String file-name]
+  (WordlistLoader/getSnowballWordSet
+   (IOUtils/getDecodingReader SnowballFilter
+                              file-name
+                              StandardCharsets/UTF_8)))
+
+(defn resource->wordset ^CharArraySet [^String resource-file-name]
+  (WordlistLoader/getSnowballWordSet
+   (IOUtils/getDecodingReader
+    (io/input-stream
+     (io/resource resource-file-name))
+    StandardCharsets/UTF_8)))
+
+(defn stream->wordset ^CharArraySet [^InputStream istream]
+  (WordlistLoader/getSnowballWordSet
+   (IOUtils/getDecodingReader istream
+                              StandardCharsets/UTF_8)))