From f98fa61a51502e84e6a410382883b8c986fa20f9 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 13:40:56 +0300 Subject: [PATCH 01/81] Add fix-BOM functions --- src/test_xml_parser/core.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 74e8f3c..1f250f9 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -48,6 +48,13 @@ result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] result)) +(defn fix-BOM [directory] + (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) + filtered-paths (for [file filtered-files] (.getAbsolutePath file)) + files (for [path filtered-paths] (slurp path)) + [grouped-data] (get-files-data files)] + grouped-data)) + (defn get-dir-by-path [path] (let [directory (clojure.java.io/file path)] (send-directory directory '("" "" "")))) From e63d7e0cb6d213cadbc04be197750d3c205b567e Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 13:53:08 +0300 Subject: [PATCH 02/81] fix --- src/test_xml_parser/core.clj | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 1f250f9..f789b8e 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -49,10 +49,12 @@ result)) (defn fix-BOM [directory] - (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) - filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (for [path filtered-paths] (slurp path)) - [grouped-data] (get-files-data files)] + (let [grouped-data "grouped-data" + ;; filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) + ;; filtered-paths (for [file filtered-files] (.getAbsolutePath file)) + ;; files (for [path filtered-paths] (slurp path)) + ;; [grouped-data] (get-files-data files) + ] grouped-data)) (defn get-dir-by-path [path] From bfa25e48394cef5be621d90a0e8ffb1544575d4f Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 13:54:27 +0300 Subject: [PATCH 03/81] fix --- src/test_xml_parser/core.clj | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index f789b8e..1f250f9 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -49,12 +49,10 @@ result)) (defn fix-BOM [directory] - (let [grouped-data "grouped-data" - ;; filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) - ;; filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - ;; files (for [path filtered-paths] (slurp path)) - ;; [grouped-data] (get-files-data files) - ] + (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) + filtered-paths (for [file filtered-files] (.getAbsolutePath file)) + files (for [path filtered-paths] (slurp path)) + [grouped-data] (get-files-data files)] grouped-data)) (defn get-dir-by-path [path] From bc20104e26106b0d642c8bf9cfc632ffd0720b8a Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 13:57:35 +0300 Subject: [PATCH 04/81] remove instead of fix --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 1f250f9..c87bc03 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -48,7 +48,7 @@ result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] result)) -(defn fix-BOM [directory] +(defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) From 38dfcd4698a16fc3564457b281369d37e247a119 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:07:20 +0300 Subject: [PATCH 05/81] get full data --- src/test_xml_parser/core.clj | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index c87bc03..e88e9d9 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -20,12 +20,19 @@ (map (fn [[k vals]] [k (first (map :attrs vals))])) (into {}))) +(defn get-full-data [arg] + (zip-str arg)) + (defn get-data [arg] (let [zip-val (zip-str arg)] (if (= (:tag (first zip-val)) :testsuites) (group-testcase-data (filter-tags (:content (first (:content (first zip-val)))) :testcase)) (group-testcase-data (filter-tags (:content (first zip-val)) :testcase))))) +(defn get-full-files-data [files] + (let [grouped-files (for [file files] (get-full-data file))] + grouped-files)) + (defn get-files-data [files] (let [grouped-files (for [file files] (get-data file))] grouped-files)) @@ -52,7 +59,7 @@ (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) - [grouped-data] (get-files-data files)] + [grouped-data] (get-full-files-data files)] grouped-data)) (defn get-dir-by-path [path] From 57319f8446178f40ef9ed0710cfad5d7882932a6 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:26:52 +0300 Subject: [PATCH 06/81] get by bytes --- src/test_xml_parser/core.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index e88e9d9..c32a4a9 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -6,6 +6,9 @@ [clojure.string :as str]) (:import [java.io File])) +(defn zip-str-bytes [s] + (xml/parse (java.io.ByteArrayInputStream. (.getBytes s)))) + (defn zip-str [s] (zip/xml-zip (xml/parse (java.io.ByteArrayInputStream. (.getBytes s))))) @@ -21,7 +24,7 @@ (into {}))) (defn get-full-data [arg] - (zip-str arg)) + (zip-str-bytes arg)) (defn get-data [arg] (let [zip-val (zip-str arg)] From 7a6f4e05feefe1d7efd64ef8b207218505eef8ee Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:28:36 +0300 Subject: [PATCH 07/81] bytes --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index c32a4a9..c9a5e9d 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -7,7 +7,7 @@ (:import [java.io File])) (defn zip-str-bytes [s] - (xml/parse (java.io.ByteArrayInputStream. (.getBytes s)))) + (java.io.ByteArrayInputStream. (.getBytes s))) (defn zip-str [s] (zip/xml-zip From c58a33ce3effd161d7793d6e3415e199eb52bab8 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:31:26 +0300 Subject: [PATCH 08/81] byte --- src/test_xml_parser/core.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index c9a5e9d..ddc6951 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -7,7 +7,8 @@ (:import [java.io File])) (defn zip-str-bytes [s] - (java.io.ByteArrayInputStream. (.getBytes s))) + (zip/xml-zip + (java.io.ByteArrayInputStream. (.getBytes s)))) (defn zip-str [s] (zip/xml-zip From eebae93a1627355221e2695ac957f24271095e63 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:34:24 +0300 Subject: [PATCH 09/81] byte --- src/test_xml_parser/core.clj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ddc6951..5714ddf 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -1,13 +1,14 @@ (ns test-xml-parser.core (:require - [clojure.xml :as xml] - [clojure.zip :as zip] + [clojure.xml :as xml] + [clojure.zip :as zip] [clojure.pprint :as pprint] - [clojure.string :as str]) + [clojure.string :as str] + [io :as io]) (:import [java.io File])) (defn zip-str-bytes [s] - (zip/xml-zip + (io/input-stream (java.io.ByteArrayInputStream. (.getBytes s)))) (defn zip-str [s] From c2b08670bd24a73cf9019717271a473d9e173cd8 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:36:46 +0300 Subject: [PATCH 10/81] byte --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 5714ddf..4f6bcab 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -4,7 +4,7 @@ [clojure.zip :as zip] [clojure.pprint :as pprint] [clojure.string :as str] - [io :as io]) + [clojure.io :as io]) (:import [java.io File])) (defn zip-str-bytes [s] From 10e13c854e3062ea3864ca0673f3f5ba9e59605d Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:39:50 +0300 Subject: [PATCH 11/81] byte --- src/test_xml_parser/core.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 4f6bcab..3b37074 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -1,10 +1,10 @@ (ns test-xml-parser.core (:require - [clojure.xml :as xml] - [clojure.zip :as zip] - [clojure.pprint :as pprint] - [clojure.string :as str] - [clojure.io :as io]) + [clojure.xml :as xml] + [clojure.zip :as zip] + [clojure.pprint :as pprint] + [clojure.string :as str] + [clojure.java.io :as io]) (:import [java.io File])) (defn zip-str-bytes [s] From 93c45d9ef63a0ebfe5826e9b6d7729b7a4a4f10f Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:44:52 +0300 Subject: [PATCH 12/81] byte --- src/test_xml_parser/core.clj | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 3b37074..a97ac81 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -7,9 +7,17 @@ [clojure.java.io :as io]) (:import [java.io File])) +(defn file->bytes [file] + (with-open [xin (io/input-stream file) + xout (java.io.ByteArrayOutputStream.)] + (io/copy xin xout) + (.toByteArray xout))) + (defn zip-str-bytes [s] - (io/input-stream - (java.io.ByteArrayInputStream. (.getBytes s)))) + (file->bytes s) + ;; (.toByteArray + ;; (java.io.ByteArrayInputStream. (.getBytes s))) + ) (defn zip-str [s] (zip/xml-zip From 2a08ebd2a70da1bbe98a8ca7bf9c80dae2c799c8 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:56:39 +0300 Subject: [PATCH 13/81] byte --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index a97ac81..bda766d 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -72,7 +72,7 @@ (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) - [grouped-data] (get-full-files-data files)] + grouped-data (get-full-files-data files)] grouped-data)) (defn get-dir-by-path [path] From 53e097261cc6141b195f94a4d2ed41493372912b Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 14:58:44 +0300 Subject: [PATCH 14/81] byte --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index bda766d..a97ac81 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -72,7 +72,7 @@ (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) - grouped-data (get-full-files-data files)] + [grouped-data] (get-full-files-data files)] grouped-data)) (defn get-dir-by-path [path] From a5ed08f0d7d6c7e8887adfc82cb53d8eef1a00ba Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 15:06:31 +0300 Subject: [PATCH 15/81] byte --- src/test_xml_parser/core.clj | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index a97ac81..b6620a6 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -14,10 +14,8 @@ (.toByteArray xout))) (defn zip-str-bytes [s] - (file->bytes s) - ;; (.toByteArray - ;; (java.io.ByteArrayInputStream. (.getBytes s))) - ) + (.toByteArray + (java.io.ByteArrayInputStream. (.getBytes s)))) (defn zip-str [s] (zip/xml-zip From ec206aeaa2daff2217d71674b4078be78e0482fc Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 15:18:03 +0300 Subject: [PATCH 16/81] byte --- src/test_xml_parser/core.clj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index b6620a6..e689158 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -14,8 +14,11 @@ (.toByteArray xout))) (defn zip-str-bytes [s] - (.toByteArray - (java.io.ByteArrayInputStream. (.getBytes s)))) + (pprint/pptint "s: ") + (pprint/pptint s) + (pprint/pptint "s") + (pprint/pptint s) + (xml/parse (java.io.ByteArrayInputStream. (.getBytes s)))) (defn zip-str [s] (zip/xml-zip @@ -32,6 +35,8 @@ (into {}))) (defn get-full-data [arg] + (pprint/pptint "arg: ") + (pprint/pptint arg) (zip-str-bytes arg)) (defn get-data [arg] From 048bee9f14a14a145006917a3b14870052dab41a Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 15:19:48 +0300 Subject: [PATCH 17/81] bytes --- src/test_xml_parser/core.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index e689158..b78c04a 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -14,10 +14,10 @@ (.toByteArray xout))) (defn zip-str-bytes [s] - (pprint/pptint "s: ") - (pprint/pptint s) - (pprint/pptint "s") - (pprint/pptint s) + (pprint/pprint "s: ") + (pprint/pprint s) + (pprint/pprint "s") + (pprint/pprint s) (xml/parse (java.io.ByteArrayInputStream. (.getBytes s)))) (defn zip-str [s] From 329b762ca60ba2f70c76440e39f47e287ca7fb43 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 15:20:59 +0300 Subject: [PATCH 18/81] pprint --- src/test_xml_parser/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index b78c04a..a7de731 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,8 +35,8 @@ (into {}))) (defn get-full-data [arg] - (pprint/pptint "arg: ") - (pprint/pptint arg) + (pprint/pprint "arg: ") + (pprint/pprint arg) (zip-str-bytes arg)) (defn get-data [arg] From b53cedee940805a13e9992641aa113bc086826d8 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 16:17:41 +0300 Subject: [PATCH 19/81] slurp-bytes --- src/test_xml_parser/core.clj | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index a7de731..e8ec695 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -39,6 +39,13 @@ (pprint/pprint arg) (zip-str-bytes arg)) +(defn slurp-bytes + "Slurp the bytes from a slurpable thing" + [x] + (with-open [out (java.io.ByteArrayOutputStream.)] + (clojure.java.io/copy (clojure.java.io/input-stream x) out) + (.toByteArray out)) + (defn get-data [arg] (let [zip-val (zip-str arg)] (if (= (:tag (first zip-val)) :testsuites) @@ -46,7 +53,7 @@ (group-testcase-data (filter-tags (:content (first zip-val)) :testcase))))) (defn get-full-files-data [files] - (let [grouped-files (for [file files] (get-full-data file))] + (let [grouped-files (for [file files] (slurp-bytes file))] grouped-files)) (defn get-files-data [files] From 93504886f14bc949022b7cdc62e3764c05f40201 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 16:19:21 +0300 Subject: [PATCH 20/81] close function --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index e8ec695..3990474 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -44,7 +44,7 @@ [x] (with-open [out (java.io.ByteArrayOutputStream.)] (clojure.java.io/copy (clojure.java.io/input-stream x) out) - (.toByteArray out)) + (.toByteArray out))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 9907945dc4d944a68cd5690a489c8cc599c26bb7 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 16:25:12 +0300 Subject: [PATCH 21/81] slurp files --- src/test_xml_parser/core.clj | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 3990474..627b770 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -52,9 +52,9 @@ (group-testcase-data (filter-tags (:content (first (:content (first zip-val)))) :testcase)) (group-testcase-data (filter-tags (:content (first zip-val)) :testcase))))) -(defn get-full-files-data [files] - (let [grouped-files (for [file files] (slurp-bytes file))] - grouped-files)) +;; (defn get-full-files-data [files] +;; (let [grouped-files (for [file files] (slurp-bytes file))] +;; grouped-files)) (defn get-files-data [files] (let [grouped-files (for [file files] (get-data file))] @@ -81,9 +81,10 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (for [path filtered-paths] (slurp path)) - [grouped-data] (get-full-files-data files)] - grouped-data)) + files (for [path filtered-paths] (slurp-bytes path)) + ;; [grouped-data] (get-full-files-data files) + ] + files)) (defn get-dir-by-path [path] (let [directory (clojure.java.io/file path)] From 3dd8a02a08a433a7f7d9c4f8e9215e504cd000b9 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 16:50:25 +0300 Subject: [PATCH 22/81] into char --- src/test_xml_parser/core.clj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 627b770..73ebb45 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -42,9 +42,10 @@ (defn slurp-bytes "Slurp the bytes from a slurpable thing" [x] - (with-open [out (java.io.ByteArrayOutputStream.)] - (clojure.java.io/copy (clojure.java.io/input-stream x) out) - (.toByteArray out))) + (map char + (with-open [out (java.io.ByteArrayOutputStream.)] + (clojure.java.io/copy (clojure.java.io/input-stream x) out) + (.toByteArray out)))) (defn get-data [arg] (let [zip-val (zip-str arg)] @@ -81,7 +82,7 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (for [path filtered-paths] (slurp-bytes path)) + [files] (for [path filtered-paths] (slurp-bytes path)) ;; [grouped-data] (get-full-files-data files) ] files)) From 1f39186127e6073fbf9dd58411f8d50242eead70 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 16:55:38 +0300 Subject: [PATCH 23/81] first char --- src/test_xml_parser/core.clj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 73ebb45..51c49db 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -42,10 +42,11 @@ (defn slurp-bytes "Slurp the bytes from a slurpable thing" [x] - (map char - (with-open [out (java.io.ByteArrayOutputStream.)] - (clojure.java.io/copy (clojure.java.io/input-stream x) out) - (.toByteArray out)))) + (first + (map char + (with-open [out (java.io.ByteArrayOutputStream.)] + (clojure.java.io/copy (clojure.java.io/input-stream x) out) + (.toByteArray out))))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 91685e5b62fb9c00bf5e8b35686e11b7bc377121 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 16:59:53 +0300 Subject: [PATCH 24/81] all results --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 51c49db..cf100c4 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -83,7 +83,7 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - [files] (for [path filtered-paths] (slurp-bytes path)) + files (for [path filtered-paths] (slurp-bytes path)) ;; [grouped-data] (get-full-files-data files) ] files)) From f527d21e0152140416c7914c9a5cf906982c7d37 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 17:08:57 +0300 Subject: [PATCH 25/81] file->bytes --- src/test_xml_parser/core.clj | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index cf100c4..6b7c70e 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -42,11 +42,13 @@ (defn slurp-bytes "Slurp the bytes from a slurpable thing" [x] - (first - (map char - (with-open [out (java.io.ByteArrayOutputStream.)] - (clojure.java.io/copy (clojure.java.io/input-stream x) out) - (.toByteArray out))))) + (file->bytes (io/file "/foo-pc" "junk.txt")) + ;; (first + ;; (map char + ;; (with-open [out (java.io.ByteArrayOutputStream.)] + ;; (clojure.java.io/copy (clojure.java.io/input-stream x) out) + ;; (.toByteArray out)))) + ) (defn get-data [arg] (let [zip-val (zip-str arg)] From 15112e4f3adaf29215e9857c518e6daa637684fc Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 17:16:59 +0300 Subject: [PATCH 26/81] files --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 6b7c70e..ea9ea6c 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -42,7 +42,7 @@ (defn slurp-bytes "Slurp the bytes from a slurpable thing" [x] - (file->bytes (io/file "/foo-pc" "junk.txt")) + (file->bytes (io/file x)) ;; (first ;; (map char ;; (with-open [out (java.io.ByteArrayOutputStream.)] From af0272d8283edcc120038388799dc5116c775cda Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 17:50:56 +0300 Subject: [PATCH 27/81] debomify --- src/test_xml_parser/core.clj | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ea9ea6c..6264d20 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -42,7 +42,8 @@ (defn slurp-bytes "Slurp the bytes from a slurpable thing" [x] - (file->bytes (io/file x)) + (first + (map char (file->bytes (io/file x)))) ;; (first ;; (map char ;; (with-open [out (java.io.ByteArrayOutputStream.)] @@ -50,6 +51,17 @@ ;; (.toByteArray out)))) ) + +(defn debomify + [^String line] + (let [bom "\uFEFF"] + (if (.startsWith line bom) + (.substring line 1) + line))) + +(defn file-bom [file] + (doall (map #(.length %) (.split (debomify (slurp file)) "\n")))) + (defn get-data [arg] (let [zip-val (zip-str arg)] (if (= (:tag (first zip-val)) :testsuites) @@ -85,7 +97,7 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (for [path filtered-paths] (slurp-bytes path)) + files (for [path filtered-paths] (file-bom path)) ;; [grouped-data] (get-full-files-data files) ] files)) From 1dcc7e9b7551ce1cd59ffd295b0359e38987c33c Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 18:10:21 +0300 Subject: [PATCH 28/81] without spliting --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 6264d20..11db181 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -60,7 +60,7 @@ line))) (defn file-bom [file] - (doall (map #(.length %) (.split (debomify (slurp file)) "\n")))) + (doall (map #(.length %) (debomify (slurp file))))) (defn get-data [arg] (let [zip-val (zip-str arg)] From dabb2dabeee7baa678f9e630d7fe0b933665c07c Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 18:12:11 +0300 Subject: [PATCH 29/81] debomify --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 11db181..f9ca7f7 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -60,7 +60,7 @@ line))) (defn file-bom [file] - (doall (map #(.length %) (debomify (slurp file))))) + (doall (map debomify (slurp file)))) (defn get-data [arg] (let [zip-val (zip-str arg)] From e2dd3f3ccf3c36389affeeac7ffa48e1f9b235ee Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 18:15:09 +0300 Subject: [PATCH 30/81] debomify --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index f9ca7f7..101cd7a 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -60,7 +60,7 @@ line))) (defn file-bom [file] - (doall (map debomify (slurp file)))) + (debomify (slurp file))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 48aa5e59a26b3bd9fbbd3ff7b4f2beaead43459f Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 19:16:35 +0300 Subject: [PATCH 31/81] display-files --- src/test_xml_parser/core.clj | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 101cd7a..e234c2e 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -102,6 +102,15 @@ ] files)) +(defn return-file [file] + (pprint/pprint (slurp file))) + +(defn return-files [directory] + (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) + filtered-paths (for [file filtered-files] (.getAbsolutePath file)) + files (for [path filtered-paths] (return-file path))] + files)) + (defn get-dir-by-path [path] (let [directory (clojure.java.io/file path)] (send-directory directory '("" "" "")))) From 3af3abb3094933a58fbbeac45f09200c900d2490 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 19:40:57 +0300 Subject: [PATCH 32/81] save domless files --- src/test_xml_parser/core.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index e234c2e..d36b545 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -60,7 +60,8 @@ line))) (defn file-bom [file] - (debomify (slurp file))) + (let [domless-file (debomify (slurp file))] + (spit domless-file file))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 017e6ce0da89c70406d79880c79be4918790d6dd Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 19:44:18 +0300 Subject: [PATCH 33/81] domless fix --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index d36b545..2cda890 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -61,7 +61,7 @@ (defn file-bom [file] (let [domless-file (debomify (slurp file))] - (spit domless-file file))) + (spit file domless-file))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 6e74e0598b5b2a612e52322b40c8b72f6f829b8e Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 8 Jul 2020 19:54:53 +0300 Subject: [PATCH 34/81] clean code --- src/test_xml_parser/core.clj | 39 +----------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 2cda890..083799f 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -7,19 +7,6 @@ [clojure.java.io :as io]) (:import [java.io File])) -(defn file->bytes [file] - (with-open [xin (io/input-stream file) - xout (java.io.ByteArrayOutputStream.)] - (io/copy xin xout) - (.toByteArray xout))) - -(defn zip-str-bytes [s] - (pprint/pprint "s: ") - (pprint/pprint s) - (pprint/pprint "s") - (pprint/pprint s) - (xml/parse (java.io.ByteArrayInputStream. (.getBytes s)))) - (defn zip-str [s] (zip/xml-zip (xml/parse (java.io.ByteArrayInputStream. (.getBytes s))))) @@ -34,24 +21,6 @@ (map (fn [[k vals]] [k (first (map :attrs vals))])) (into {}))) -(defn get-full-data [arg] - (pprint/pprint "arg: ") - (pprint/pprint arg) - (zip-str-bytes arg)) - -(defn slurp-bytes - "Slurp the bytes from a slurpable thing" - [x] - (first - (map char (file->bytes (io/file x)))) - ;; (first - ;; (map char - ;; (with-open [out (java.io.ByteArrayOutputStream.)] - ;; (clojure.java.io/copy (clojure.java.io/input-stream x) out) - ;; (.toByteArray out)))) - ) - - (defn debomify [^String line] (let [bom "\uFEFF"] @@ -69,10 +38,6 @@ (group-testcase-data (filter-tags (:content (first (:content (first zip-val)))) :testcase)) (group-testcase-data (filter-tags (:content (first zip-val)) :testcase))))) -;; (defn get-full-files-data [files] -;; (let [grouped-files (for [file files] (slurp-bytes file))] -;; grouped-files)) - (defn get-files-data [files] (let [grouped-files (for [file files] (get-data file))] grouped-files)) @@ -98,9 +63,7 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (for [path filtered-paths] (file-bom path)) - ;; [grouped-data] (get-full-files-data files) - ] + files (for [path filtered-paths] (file-bom path))] files)) (defn return-file [file] From e96584d555ef7aa3d5ab62a4a046f063dd936ca9 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Thu, 9 Jul 2020 16:43:39 +0300 Subject: [PATCH 35/81] with clj-bom --- project.clj | 3 ++- src/test_xml_parser/core.clj | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index af2b24e..dacaff2 100644 --- a/project.clj +++ b/project.clj @@ -4,6 +4,7 @@ :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.0"] - [org.flatland/useful "0.9.0"]] + [org.flatland/useful "0.9.0"] + [clj-bom "0.1.2"]] :main test-xml-parser.core :repl-options {:init-ns test-xml-parser.core}) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 083799f..e9a3519 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -4,7 +4,8 @@ [clojure.zip :as zip] [clojure.pprint :as pprint] [clojure.string :as str] - [clojure.java.io :as io]) + [clojure.java.io :as io] + [clj-bom.core :as bom]) (:import [java.io File])) (defn zip-str [s] From dfe8db00368233042adfdf25c53bb07dddcfe6c3 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Thu, 9 Jul 2020 17:23:30 +0300 Subject: [PATCH 36/81] print is bom --- project.clj | 3 ++- src/test_xml_parser/core.clj | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index dacaff2..d0cc59d 100644 --- a/project.clj +++ b/project.clj @@ -5,6 +5,7 @@ :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.0"] [org.flatland/useful "0.9.0"] - [clj-bom "0.1.2"]] + ;; [org.clojure/clj-bom "0.1.2"] + ] :main test-xml-parser.core :repl-options {:init-ns test-xml-parser.core}) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index e9a3519..ebd5994 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -5,7 +5,8 @@ [clojure.pprint :as pprint] [clojure.string :as str] [clojure.java.io :as io] - [clj-bom.core :as bom]) + ;; [clj-bom.core :as bom] + ) (:import [java.io File])) (defn zip-str [s] @@ -26,7 +27,9 @@ [^String line] (let [bom "\uFEFF"] (if (.startsWith line bom) - (.substring line 1) + (doall + (pprint/pprint "IS BOM") + (.substring line 1)) line))) (defn file-bom [file] @@ -64,6 +67,7 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) + _ (pprint/pprint "IN HERE") files (for [path filtered-paths] (file-bom path))] files)) From d7b62d9fd57bad262c5a43ea8875354a50965c10 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Thu, 9 Jul 2020 17:24:55 +0300 Subject: [PATCH 37/81] printline --- src/test_xml_parser/core.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ebd5994..8c510e3 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -25,7 +25,8 @@ (defn debomify [^String line] - (let [bom "\uFEFF"] + (let [bom "\uFEFF" + _ (pprint/pprint {"line:" line})] (if (.startsWith line bom) (doall (pprint/pprint "IS BOM") From 9247b46de1bff00fb30f2d67c3e4a807555dc35d Mon Sep 17 00:00:00 2001 From: shmuelko Date: Thu, 9 Jul 2020 17:37:15 +0300 Subject: [PATCH 38/81] ppint --- src/test_xml_parser/core.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 8c510e3..1ce02d3 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -34,7 +34,8 @@ line))) (defn file-bom [file] - (let [domless-file (debomify (slurp file))] + (let [domless-file (debomify (slurp file)) + _ (pprint/pprint {"file: " file})] (spit file domless-file))) (defn get-data [arg] From 28f873a134b94521b753a109e2b84da45b43467b Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 09:09:41 +0300 Subject: [PATCH 39/81] doall --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 1ce02d3..74545f1 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -70,7 +70,7 @@ (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) _ (pprint/pprint "IN HERE") - files (for [path filtered-paths] (file-bom path))] + files (doall (map file-bom filtered-paths))] files)) (defn return-file [file] From dcc630bb7721b0fe976132d0e1f5a909f96e0cb0 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 09:11:26 +0300 Subject: [PATCH 40/81] doall --- src/test_xml_parser/core.clj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 74545f1..d178191 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -28,9 +28,7 @@ (let [bom "\uFEFF" _ (pprint/pprint {"line:" line})] (if (.startsWith line bom) - (doall - (pprint/pprint "IS BOM") - (.substring line 1)) + (.substring line 1) line))) (defn file-bom [file] From e223516ba21d48c1ca674ca6abf63b0e8ee88c95 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 09:18:59 +0300 Subject: [PATCH 41/81] remove prints --- src/test_xml_parser/core.clj | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index d178191..481fe71 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -25,15 +25,13 @@ (defn debomify [^String line] - (let [bom "\uFEFF" - _ (pprint/pprint {"line:" line})] + (let [bom "\uFEFF"] (if (.startsWith line bom) (.substring line 1) line))) (defn file-bom [file] - (let [domless-file (debomify (slurp file)) - _ (pprint/pprint {"file: " file})] + (let [domless-file (debomify (slurp file))] (spit file domless-file))) (defn get-data [arg] @@ -67,7 +65,6 @@ (defn remove-bom [directory] (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - _ (pprint/pprint "IN HERE") files (doall (map file-bom filtered-paths))] files)) From 016435a8699a5c5765c3ed4d53243ec7180b3eac Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 12:19:46 +0300 Subject: [PATCH 42/81] save bomless files into tmp folder --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 481fe71..fca526e 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -32,7 +32,7 @@ (defn file-bom [file] (let [domless-file (debomify (slurp file))] - (spit file domless-file))) + (spit (str "tmp/"file) domless-file))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 9c05fb73838b7c38f832809fb5180af5654c7a90 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 12:32:11 +0300 Subject: [PATCH 43/81] add new files to tmp --- src/test_xml_parser/core.clj | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index fca526e..d187027 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -31,8 +31,12 @@ line))) (defn file-bom [file] - (let [domless-file (debomify (slurp file))] - (spit (str "tmp/"file) domless-file))) + (let [domless-file (debomify (slurp file)) + full-path (str/split file #"/") + directory (butlast full-path) + filename (last full-path) + new-path (str directory "tmp/" filename)] + (spit new-path domless-file))) (defn get-data [arg] (let [zip-val (zip-str arg)] From 006eed3a4959171fad8e937a83ae98bd7d9f3a54 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 12:35:43 +0300 Subject: [PATCH 44/81] join folder path --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index d187027..845470e 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -33,7 +33,7 @@ (defn file-bom [file] (let [domless-file (debomify (slurp file)) full-path (str/split file #"/") - directory (butlast full-path) + directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "tmp/" filename)] (spit new-path domless-file))) From 19fb7a08e9827b4c70be8fc28a15c6066d31c902 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 12:38:49 +0300 Subject: [PATCH 45/81] fix --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 845470e..0fe13ae 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,7 +35,7 @@ full-path (str/split file #"/") directory (str/join "/" (butlast full-path)) filename (last full-path) - new-path (str directory "tmp/" filename)] + new-path (str directory "/tmp/" filename)] (spit new-path domless-file))) (defn get-data [arg] From bb0dd614cce551bd8edd6669345030bbf70ca53d Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 13:40:59 +0300 Subject: [PATCH 46/81] make tmp dir --- src/test_xml_parser/core.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 0fe13ae..3732f36 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -5,7 +5,6 @@ [clojure.pprint :as pprint] [clojure.string :as str] [clojure.java.io :as io] - ;; [clj-bom.core :as bom] ) (:import [java.io File])) @@ -36,6 +35,8 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] + (io/make-parents new-path) + ;; (.mkdir (java.io.File. (str directory "/tmp"))) (spit new-path domless-file))) (defn get-data [arg] From 9466af3b1e5c698849f0a5acb97198481128df0b Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 14:28:33 +0300 Subject: [PATCH 47/81] run over all folders --- src/test_xml_parser/core.clj | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 3732f36..7e5d741 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,8 +35,7 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] - (io/make-parents new-path) - ;; (.mkdir (java.io.File. (str directory "/tmp"))) + (.mkdir (java.io.File. (str directory "/tmp"))) (spit new-path domless-file))) (defn get-data [arg] @@ -60,11 +59,11 @@ merge-content)) (defn send-directory [directory parsed-content] - (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) + (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq (io/file directory))) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) - result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] + result (parse-n-merge-data grouped-data parsed-content)] result)) (defn remove-bom [directory] From 6e270732629d67e935200b40060aa430fec5c2d3 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 14:36:17 +0300 Subject: [PATCH 48/81] get file --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 7e5d741..8b6fcfc 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -67,7 +67,7 @@ result)) (defn remove-bom [directory] - (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq directory)) + (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq (io/file directory))) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (doall (map file-bom filtered-paths))] files)) From 4cbf6e332683856b6e24a77060f45b7a32601a90 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 14:48:26 +0300 Subject: [PATCH 49/81] directory --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 8b6fcfc..ea95277 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -59,7 +59,7 @@ merge-content)) (defn send-directory [directory parsed-content] - (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq (io/file directory))) + (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (io/file directory)) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) From 02918d32f75bdf8c3b7372f2eaf0a7fb07ee6504 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 14:52:24 +0300 Subject: [PATCH 50/81] debug prints --- src/test_xml_parser/core.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ea95277..ac4a8ea 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -59,7 +59,10 @@ merge-content)) (defn send-directory [directory parsed-content] - (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (io/file directory)) + (let [_ (pprint/pprint {"directory: " directory}) + _ (pprint/pprint {"parsed-content: " parsed-content}) + file-seqs (file-seq (io/file directory)) + filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) From 9a0ae9bed0863fe082f074a1a0182a82e5f3208e Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 15:25:40 +0300 Subject: [PATCH 51/81] clean debugs prints --- src/test_xml_parser/core.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ac4a8ea..c62644f 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -59,18 +59,17 @@ merge-content)) (defn send-directory [directory parsed-content] - (let [_ (pprint/pprint {"directory: " directory}) - _ (pprint/pprint {"parsed-content: " parsed-content}) - file-seqs (file-seq (io/file directory)) + (let [file-seqs (file-seq (io/file directory)) filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) - result (parse-n-merge-data grouped-data parsed-content)] + result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] result)) (defn remove-bom [directory] - (let [filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) (file-seq (io/file directory))) + (let [file-seqs (file-seq (io/file directory)) + filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (doall (map file-bom filtered-paths))] files)) From 3d87bca2ed4f5bbfe32e3e4c2c516772820965ee Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 15:28:11 +0300 Subject: [PATCH 52/81] remove previus tmp folde --- src/test_xml_parser/core.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index c62644f..ddf138b 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,6 +35,7 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] + (io/delete-file (str directory "/tmp")) (.mkdir (java.io.File. (str directory "/tmp"))) (spit new-path domless-file))) From aa06cbb2887cdefe50a84163b41b20861e508ec7 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 15:32:21 +0300 Subject: [PATCH 53/81] create tmp only if exsist --- src/test_xml_parser/core.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ddf138b..d0727b3 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,8 +35,7 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] - (io/delete-file (str directory "/tmp")) - (.mkdir (java.io.File. (str directory "/tmp"))) + (when (not (.isDirectory (io/file (str directory "/tmp")))) (.mkdir (java.io.File. (str directory "/tmp")))) (spit new-path domless-file))) (defn get-data [arg] From b284929327aec8691a04df119c338aa10d95014d Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 15:35:30 +0300 Subject: [PATCH 54/81] exists? --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index d0727b3..4c4b873 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,7 +35,7 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] - (when (not (.isDirectory (io/file (str directory "/tmp")))) (.mkdir (java.io.File. (str directory "/tmp")))) + (when (not (.exists (io/file (str directory "/tmp")))) (.mkdir (java.io.File. (str directory "/tmp")))) (spit new-path domless-file))) (defn get-data [arg] From 522f45c571bf0ccb86d1de0515ca1f718ed1c374 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 15:41:22 +0300 Subject: [PATCH 55/81] debug check why stil create fodler tmp --- src/test_xml_parser/core.clj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 4c4b873..ba717da 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,6 +35,10 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] + (pprint/pprint {"new-path: " new-path}) + (pprint/pprint {"(str directory \"/tmp\"): " (str directory "/tmp")}) + (pprint/pprint {"(.exists (io/file (str directory \"/tmp\"))): " (.exists (io/file (str directory "/tmp")))}) + (pprint/pprint {"(.isDirectory (io/file (str directory \"/tmp\"))): " (.isDirectory (io/file (str directory "/tmp")))}) (when (not (.exists (io/file (str directory "/tmp")))) (.mkdir (java.io.File. (str directory "/tmp")))) (spit new-path domless-file))) From 7f7ef50ef7e269942451b144545613f26f8cd0fa Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 15:45:36 +0300 Subject: [PATCH 56/81] more debug, I see too many files included after filter --- src/test_xml_parser/core.clj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ba717da..230c8a3 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -72,10 +72,15 @@ result)) (defn remove-bom [directory] - (let [file-seqs (file-seq (io/file directory)) + (let [_ (pprint/pprint {"directory: " directory}) + file-seqs (file-seq (io/file directory)) + _ (pprint/pprint {"file-seqs: " file-seqs}) filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + _ (pprint/pprint {"filtered-files: " filtered-files}) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (doall (map file-bom filtered-paths))] + _ (pprint/pprint {"filtered-paths: " filtered-paths}) + files (doall (map file-bom filtered-paths)) + ] files)) (defn return-file [file] From 6f70e420048dc96230b631ae70eaebd5ac929741 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 16:04:48 +0300 Subject: [PATCH 57/81] only this folder and not sup folders --- src/test_xml_parser/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 230c8a3..e79cd7c 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -64,7 +64,7 @@ (defn send-directory [directory parsed-content] (let [file-seqs (file-seq (io/file directory)) - filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + filtered-files (filter (fn [file] (and (str/ends-with? (.getAbsolutePath file) ".xml") (not ((str/includes? file "/tmp"))))) file-seqs) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) @@ -75,7 +75,7 @@ (let [_ (pprint/pprint {"directory: " directory}) file-seqs (file-seq (io/file directory)) _ (pprint/pprint {"file-seqs: " file-seqs}) - filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + filtered-files (filter (fn [file] (and (str/ends-with? (.getAbsolutePath file) ".xml") (not ((str/includes? file "/tmp"))))) file-seqs) _ (pprint/pprint {"filtered-files: " filtered-files}) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) _ (pprint/pprint {"filtered-paths: " filtered-paths}) From fe6e7346207c00cd629c59073450758658784a08 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 16:09:22 +0300 Subject: [PATCH 58/81] not contains --- src/test_xml_parser/core.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index e79cd7c..419c93e 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -64,7 +64,7 @@ (defn send-directory [directory parsed-content] (let [file-seqs (file-seq (io/file directory)) - filtered-files (filter (fn [file] (and (str/ends-with? (.getAbsolutePath file) ".xml") (not ((str/includes? file "/tmp"))))) file-seqs) + filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) @@ -75,7 +75,9 @@ (let [_ (pprint/pprint {"directory: " directory}) file-seqs (file-seq (io/file directory)) _ (pprint/pprint {"file-seqs: " file-seqs}) - filtered-files (filter (fn [file] (and (str/ends-with? (.getAbsolutePath file) ".xml") (not ((str/includes? file "/tmp"))))) file-seqs) + xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + _ (pprint/pprint {"xml-files: " xml-files}) + filtered-files (filter (fn [file] (not (str/contains? file "/tmp"))) xml-files) _ (pprint/pprint {"filtered-files: " filtered-files}) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) _ (pprint/pprint {"filtered-paths: " filtered-paths}) From bdfc3fe375697bc434e220803fa43728d17b3ba3 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 16:15:39 +0300 Subject: [PATCH 59/81] filters --- src/test_xml_parser/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 419c93e..4d4dc61 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -77,10 +77,10 @@ _ (pprint/pprint {"file-seqs: " file-seqs}) xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) _ (pprint/pprint {"xml-files: " xml-files}) - filtered-files (filter (fn [file] (not (str/contains? file "/tmp"))) xml-files) + filtered-files (filter (fn [file] (not (str/contains? file "/tmp/"))) xml-files) _ (pprint/pprint {"filtered-files: " filtered-files}) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - _ (pprint/pprint {"filtered-paths: " filtered-paths}) + _ (pprint/pprint {"filtered-paths:" filtered-paths}) files (doall (map file-bom filtered-paths)) ] files)) From 7930074bbcfa61de265605f7d4551aaee1642d02 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 16:17:20 +0300 Subject: [PATCH 60/81] includes --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 4d4dc61..ac7010d 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -77,7 +77,7 @@ _ (pprint/pprint {"file-seqs: " file-seqs}) xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) _ (pprint/pprint {"xml-files: " xml-files}) - filtered-files (filter (fn [file] (not (str/contains? file "/tmp/"))) xml-files) + filtered-files (filter (fn [file] (not (str/includes? file "/tmp/"))) xml-files) _ (pprint/pprint {"filtered-files: " filtered-files}) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) _ (pprint/pprint {"filtered-paths:" filtered-paths}) From 2bf49aa7edce6f1646d244eff0e897357f219048 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 16:19:32 +0300 Subject: [PATCH 61/81] clean code --- src/test_xml_parser/core.clj | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ac7010d..4006cd8 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -64,7 +64,8 @@ (defn send-directory [directory parsed-content] (let [file-seqs (file-seq (io/file directory)) - filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + filtered-files (filter (fn [file] (not (str/includes? file "/tmp/"))) xml-files) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) @@ -72,15 +73,10 @@ result)) (defn remove-bom [directory] - (let [_ (pprint/pprint {"directory: " directory}) - file-seqs (file-seq (io/file directory)) - _ (pprint/pprint {"file-seqs: " file-seqs}) - xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) - _ (pprint/pprint {"xml-files: " xml-files}) + (let [file-seqs (file-seq (io/file directory)) + xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) filtered-files (filter (fn [file] (not (str/includes? file "/tmp/"))) xml-files) - _ (pprint/pprint {"filtered-files: " filtered-files}) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - _ (pprint/pprint {"filtered-paths:" filtered-paths}) files (doall (map file-bom filtered-paths)) ] files)) From 468148d7bca61d91a64f19a3c95594964b429737 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 16:24:04 +0300 Subject: [PATCH 62/81] clean debugs --- src/test_xml_parser/core.clj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 4006cd8..43b5098 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,10 +35,6 @@ directory (str/join "/" (butlast full-path)) filename (last full-path) new-path (str directory "/tmp/" filename)] - (pprint/pprint {"new-path: " new-path}) - (pprint/pprint {"(str directory \"/tmp\"): " (str directory "/tmp")}) - (pprint/pprint {"(.exists (io/file (str directory \"/tmp\"))): " (.exists (io/file (str directory "/tmp")))}) - (pprint/pprint {"(.isDirectory (io/file (str directory \"/tmp\"))): " (.isDirectory (io/file (str directory "/tmp")))}) (when (not (.exists (io/file (str directory "/tmp")))) (.mkdir (java.io.File. (str directory "/tmp")))) (spit new-path domless-file))) From 1fd1ce8130019ca466a7352e1c7febae7e960db2 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 18:36:33 +0300 Subject: [PATCH 63/81] lazy seq unlazy --- src/test_xml_parser/core.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 43b5098..333d79c 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -64,8 +64,9 @@ filtered-files (filter (fn [file] (not (str/includes? file "/tmp/"))) xml-files) filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) - [grouped-data] (get-files-data files) - result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] + [grouped-data] (get-files-data files) + result (doall (map parse-n-merge-data parsed-content grouped-data)) + ] result)) (defn remove-bom [directory] From f19453fa81c84db1b635b13b69c4288f59d02a11 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 18:38:26 +0300 Subject: [PATCH 64/81] unlazy --- src/test_xml_parser/core.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 333d79c..83fc773 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -65,8 +65,7 @@ filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) - result (doall (map parse-n-merge-data parsed-content grouped-data)) - ] + result (doall (map parse-n-merge-data parsed-content grouped-data))] result)) (defn remove-bom [directory] From d3d0b193b95c9df7a1a4cb7f1abd96ec7457ea16 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Sun, 12 Jul 2020 18:53:13 +0300 Subject: [PATCH 65/81] send-dir --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 83fc773..3645a6c 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -65,7 +65,7 @@ filtered-paths (for [file filtered-files] (.getAbsolutePath file)) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) - result (doall (map parse-n-merge-data parsed-content grouped-data))] + result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] result)) (defn remove-bom [directory] From 1ba156b66692c038f9c236aae199baa84e3fa35a Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 17:24:06 +0300 Subject: [PATCH 66/81] change seperator depends on the system --- project.clj | 4 +--- src/test_xml_parser/core.clj | 38 ++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/project.clj b/project.clj index d0cc59d..af2b24e 100644 --- a/project.clj +++ b/project.clj @@ -4,8 +4,6 @@ :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[org.clojure/clojure "1.10.0"] - [org.flatland/useful "0.9.0"] - ;; [org.clojure/clj-bom "0.1.2"] - ] + [org.flatland/useful "0.9.0"]] :main test-xml-parser.core :repl-options {:init-ns test-xml-parser.core}) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 3645a6c..748fde1 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -30,13 +30,18 @@ line))) (defn file-bom [file] - (let [domless-file (debomify (slurp file)) - full-path (str/split file #"/") - directory (str/join "/" (butlast full-path)) - filename (last full-path) - new-path (str directory "/tmp/" filename)] - (when (not (.exists (io/file (str directory "/tmp")))) (.mkdir (java.io.File. (str directory "/tmp")))) - (spit new-path domless-file))) + (let [bomless-file (debomify (slurp file)) + directory (.getParent file) + directory-name (.getName directory) + directory-parent (.getParent directory) + filename (.getName file) + seperator (if (str/includes file "/") "/" "\\") + new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] + (when (not + (.exists (io/file (str directory-parent seperator "tmp")))) + (do (.mkdir (io/file (str directory-parent seperator "tmp"))) + (.mkdir (io/file (str directory-parent seperator "tmp" seperator directory-name))))) + (spit new-path bomless-file))) (defn get-data [arg] (let [zip-val (zip-str arg)] @@ -58,23 +63,22 @@ (let [merge-content (merge (get grouped-files-map (:name parsed-content)) parsed-content)] merge-content)) +(defn get-files-path [directory] + (let [file-seqs (.listFiles (io/file directory)) + filtered-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) + filtered-paths (for [file filtered-files] (.getAbsolutePath file))] + filtered-paths)) + (defn send-directory [directory parsed-content] - (let [file-seqs (file-seq (io/file directory)) - xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) - filtered-files (filter (fn [file] (not (str/includes? file "/tmp/"))) xml-files) - filtered-paths (for [file filtered-files] (.getAbsolutePath file)) + (let [filtered-paths (get-files-path directory) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] result)) (defn remove-bom [directory] - (let [file-seqs (file-seq (io/file directory)) - xml-files (filter (fn [file] (str/ends-with? (.getAbsolutePath file) ".xml")) file-seqs) - filtered-files (filter (fn [file] (not (str/includes? file "/tmp/"))) xml-files) - filtered-paths (for [file filtered-files] (.getAbsolutePath file)) - files (doall (map file-bom filtered-paths)) - ] + (let [filtered-paths (get-files-path directory) + files (doall (map file-bom filtered-paths))] files)) (defn return-file [file] From 74a97b0ca9d8c51dec1531ad03948c0cfe3f6812 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 18:10:23 +0300 Subject: [PATCH 67/81] io/files --- src/test_xml_parser/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 748fde1..815b024 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -32,8 +32,8 @@ (defn file-bom [file] (let [bomless-file (debomify (slurp file)) directory (.getParent file) - directory-name (.getName directory) - directory-parent (.getParent directory) + directory-name (.getName (io/file directory)) + directory-parent (.getParent (io/file directory)) filename (.getName file) seperator (if (str/includes file "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] From e87a4168a0ef57c0f84b27b67c0b3a116363338d Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 18:13:59 +0300 Subject: [PATCH 68/81] parent --- src/test_xml_parser/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 815b024..0240368 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -31,9 +31,9 @@ (defn file-bom [file] (let [bomless-file (debomify (slurp file)) - directory (.getParent file) + directory (.parent file) directory-name (.getName (io/file directory)) - directory-parent (.getParent (io/file directory)) + directory-parent (.parent (io/file directory)) filename (.getName file) seperator (if (str/includes file "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] From b0e0dc564a9488315165c1f6c9374bdf96e76238 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 19:18:18 +0300 Subject: [PATCH 69/81] tmp as parent --- src/test_xml_parser/core.clj | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 0240368..b46b72a 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -29,18 +29,19 @@ (.substring line 1) line))) -(defn file-bom [file] - (let [bomless-file (debomify (slurp file)) - directory (.parent file) - directory-name (.getName (io/file directory)) - directory-parent (.parent (io/file directory)) - filename (.getName file) - seperator (if (str/includes file "/") "/" "\\") +(defn file-bom [path] + (let [bomless-file (debomify (slurp path)) + directory (.getParent (io/file path)) + directory-name (.getName (io/file directory)) + directory-parent (.getParent (io/file directory)) + filename (.getName (io/file path)) + seperator (if (str/includes path "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] - (when (not - (.exists (io/file (str directory-parent seperator "tmp")))) - (do (.mkdir (io/file (str directory-parent seperator "tmp"))) - (.mkdir (io/file (str directory-parent seperator "tmp" seperator directory-name))))) + (when + (not + (.exists (io/file (str directory-parent seperator "tmp")))) + (do (.mkdir (io/file (str directory-parent seperator "tmp"))) + (.mkdir (io/file (str directory-parent seperator "tmp" seperator directory-name))))) (spit new-path bomless-file))) (defn get-data [arg] @@ -77,9 +78,9 @@ result)) (defn remove-bom [directory] - (let [filtered-paths (get-files-path directory) - files (doall (map file-bom filtered-paths))] - files)) + (let [filtered-paths (get-files-path directory)] + (doseq [path filtered-paths] + (file-bom path)))) (defn return-file [file] (pprint/pprint (slurp file))) From 75e548d3f6fdd879ff4795e3dcf6d76c6ff356f0 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 19:25:06 +0300 Subject: [PATCH 70/81] prints --- src/test_xml_parser/core.clj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index b46b72a..26cb4f8 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -37,6 +37,12 @@ filename (.getName (io/file path)) seperator (if (str/includes path "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] + (pprint/pprint "new-path: ") + (pprint/pprint new-path) + (pprint/pprint "directory: ") + (pprint/pprint directory) + (pprint/pprint "directory-parent: ") + (pprint/pprint directory-parent) (when (not (.exists (io/file (str directory-parent seperator "tmp")))) From 23f06ab0f9748e7890203c7da174d33dceb73fb1 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 19:32:18 +0300 Subject: [PATCH 71/81] print in remove-bom --- src/test_xml_parser/core.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 26cb4f8..f017880 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -85,6 +85,9 @@ (defn remove-bom [directory] (let [filtered-paths (get-files-path directory)] + (pprint/pprint "remove-bom") + (pprint/pprint "filtered-paths: ") + (pprint/pprint filtered-paths) (doseq [path filtered-paths] (file-bom path)))) From 7f71cbb93e0852300a138b899c9a07eb98ea8109 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 20:01:43 +0300 Subject: [PATCH 72/81] includes --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index f017880..b2637e0 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -35,7 +35,7 @@ directory-name (.getName (io/file directory)) directory-parent (.getParent (io/file directory)) filename (.getName (io/file path)) - seperator (if (str/includes path "/") "/" "\\") + seperator (if (str/includes? path "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] (pprint/pprint "new-path: ") (pprint/pprint new-path) From fc56591c716547ba778528da19aa0164bf6c20f6 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 20:06:43 +0300 Subject: [PATCH 73/81] remove print --- src/test_xml_parser/core.clj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index b2637e0..469f7ef 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -37,12 +37,6 @@ filename (.getName (io/file path)) seperator (if (str/includes? path "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] - (pprint/pprint "new-path: ") - (pprint/pprint new-path) - (pprint/pprint "directory: ") - (pprint/pprint directory) - (pprint/pprint "directory-parent: ") - (pprint/pprint directory-parent) (when (not (.exists (io/file (str directory-parent seperator "tmp")))) From 6b5ad41fd50c2cb99f4b0c2871c1be256112f156 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Mon, 13 Jul 2020 20:57:37 +0300 Subject: [PATCH 74/81] remove print --- src/test_xml_parser/core.clj | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 469f7ef..2987f62 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -79,9 +79,6 @@ (defn remove-bom [directory] (let [filtered-paths (get-files-path directory)] - (pprint/pprint "remove-bom") - (pprint/pprint "filtered-paths: ") - (pprint/pprint filtered-paths) (doseq [path filtered-paths] (file-bom path)))) From c4dbefb5c374ec00c76270a91ec3fda68eaabd2b Mon Sep 17 00:00:00 2001 From: shmuelko Date: Tue, 14 Jul 2020 09:09:30 +0300 Subject: [PATCH 75/81] create inner folder is not exist --- src/test_xml_parser/core.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 2987f62..ad43eb3 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -38,10 +38,10 @@ seperator (if (str/includes? path "/") "/" "\\") new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] (when - (not - (.exists (io/file (str directory-parent seperator "tmp")))) - (do (.mkdir (io/file (str directory-parent seperator "tmp"))) - (.mkdir (io/file (str directory-parent seperator "tmp" seperator directory-name))))) + (not (.exists (io/file (str directory-parent seperator "tmp")))) + (.mkdir (io/file (str directory-parent seperator "tmp")))) + (when (not (.exists (io/file (str directory-parent seperator "tmp" seperator directory-name)))) + (.mkdir (io/file (str directory-parent seperator "tmp" seperator directory-name)))) (spit new-path bomless-file))) (defn get-data [arg] From ed0c17b3efa20b43cd17ff18a7308aae0d349125 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Tue, 14 Jul 2020 09:36:53 +0300 Subject: [PATCH 76/81] add delete-recursively --- src/test_xml_parser/core.clj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index ad43eb3..4b21130 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -8,6 +8,14 @@ ) (:import [java.io File])) +(defn delete-recursively [fname] + (let [func (fn [func f] + (when (.isDirectory f) + (doseq [f2 (.listFiles f)] + (func func f2))) + (clojure.java.io/delete-file f))] + (func func (clojure.java.io/file fname)))) + (defn zip-str [s] (zip/xml-zip (xml/parse (java.io.ByteArrayInputStream. (.getBytes s))))) From 19da6ff4c0b0dd75ebf173800afdf8dedb904f52 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Tue, 14 Jul 2020 10:06:13 +0300 Subject: [PATCH 77/81] shorten clojure.java.io --- src/test_xml_parser/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 4b21130..d4f7ae7 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -13,8 +13,8 @@ (when (.isDirectory f) (doseq [f2 (.listFiles f)] (func func f2))) - (clojure.java.io/delete-file f))] - (func func (clojure.java.io/file fname)))) + (io/delete-file f))] + (func func (io/file fname)))) (defn zip-str [s] (zip/xml-zip From 25b422a962c0f12864aabd6261a030cc3022fcd0 Mon Sep 17 00:00:00 2001 From: shmuelko Date: Tue, 14 Jul 2020 11:20:59 +0300 Subject: [PATCH 78/81] delete-recursively! --- src/test_xml_parser/core.clj | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index d4f7ae7..7dacfb5 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -8,13 +8,12 @@ ) (:import [java.io File])) -(defn delete-recursively [fname] - (let [func (fn [func f] - (when (.isDirectory f) - (doseq [f2 (.listFiles f)] - (func func f2))) - (io/delete-file f))] - (func func (io/file fname)))) +(defn delete-recursively! [fname] + (let [f (io/file fname)] + (when (.isDirectory f) + (doseq [child-path (.listFiles f)] + (delete-recursively! child-path))) + (io/delete-file f))) (defn zip-str [s] (zip/xml-zip @@ -37,19 +36,22 @@ (.substring line 1) line))) +(defn create-container-folders [directory-parent directory-name] + (when + (not (.exists (io/file directory-parent "tmp"))) + (.mkdir (io/file directory-parent "tmp"))) + (when (not (.exists (io/file directory-parent "tmp" directory-name))) + (.mkdir (io/file directory-parent "tmp" directory-name)))) + (defn file-bom [path] (let [bomless-file (debomify (slurp path)) directory (.getParent (io/file path)) directory-name (.getName (io/file directory)) directory-parent (.getParent (io/file directory)) - filename (.getName (io/file path)) - seperator (if (str/includes? path "/") "/" "\\") - new-path (str directory-parent seperator "tmp" seperator directory-name seperator filename)] - (when - (not (.exists (io/file (str directory-parent seperator "tmp")))) - (.mkdir (io/file (str directory-parent seperator "tmp")))) - (when (not (.exists (io/file (str directory-parent seperator "tmp" seperator directory-name)))) - (.mkdir (io/file (str directory-parent seperator "tmp" seperator directory-name)))) + filename (.getName (io/file path)) + new-file (io/file directory-parent "tmp" directory-name filename) + new-path (.getAbsolutePath new-file)] + (create-container-folders directory-parent directory-name) (spit new-path bomless-file))) (defn get-data [arg] From 6e44d675a4d7989772ecea160d5fa36fb407327f Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 15 Jul 2020 10:15:23 +0300 Subject: [PATCH 79/81] merge results for several folders --- src/test_xml_parser/core.clj | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 7dacfb5..a52839a 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -80,13 +80,22 @@ filtered-paths (for [file filtered-files] (.getAbsolutePath file))] filtered-paths)) -(defn send-directory [directory parsed-content] +(defn merge-results [directory parsed-content] (let [filtered-paths (get-files-path directory) files (for [path filtered-paths] (slurp path)) [grouped-data] (get-files-data files) result (for [parsed parsed-content] (parse-n-merge-data grouped-data parsed))] result)) +(defn send-directory [directory parsed-content] + (let [report-result (list ) + result (first + (conj report-result + (first + (for [dir directory] + (merge-results dir parsed-content)))))] + result)) + (defn remove-bom [directory] (let [filtered-paths (get-files-path directory)] (doseq [path filtered-paths] From e2c72b075229aa586e89d9e7fa606cdf697b3bdd Mon Sep 17 00:00:00 2001 From: shmuelko Date: Wed, 16 Sep 2020 09:07:34 +0300 Subject: [PATCH 80/81] use tmp param to set tmp folder --- src/test_xml_parser/core.clj | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index a52839a..26f0136 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -36,22 +36,22 @@ (.substring line 1) line))) -(defn create-container-folders [directory-parent directory-name] +(defn create-container-folders [directory-parent directory-name tmp] (when - (not (.exists (io/file directory-parent "tmp"))) - (.mkdir (io/file directory-parent "tmp"))) - (when (not (.exists (io/file directory-parent "tmp" directory-name))) - (.mkdir (io/file directory-parent "tmp" directory-name)))) + (not (.exists (io/file directory-parent tmp))) + (.mkdir (io/file directory-parent tmp))) + (when (not (.exists (io/file directory-parent tmp directory-name))) + (.mkdir (io/file directory-parent tmp directory-name)))) -(defn file-bom [path] +(defn file-bom [path tmp] (let [bomless-file (debomify (slurp path)) directory (.getParent (io/file path)) directory-name (.getName (io/file directory)) directory-parent (.getParent (io/file directory)) filename (.getName (io/file path)) - new-file (io/file directory-parent "tmp" directory-name filename) + new-file (io/file directory-parent tmp directory-name filename) new-path (.getAbsolutePath new-file)] - (create-container-folders directory-parent directory-name) + (create-container-folders directory-parent directory-name tmp) (spit new-path bomless-file))) (defn get-data [arg] @@ -96,10 +96,10 @@ (merge-results dir parsed-content)))))] result)) -(defn remove-bom [directory] +(defn remove-bom [directory tmp] (let [filtered-paths (get-files-path directory)] (doseq [path filtered-paths] - (file-bom path)))) + (file-bom path tmp)))) (defn return-file [file] (pprint/pprint (slurp file))) From 342e448009ffb29ed152ea80f02ac803781f3b09 Mon Sep 17 00:00:00 2001 From: stask Date: Tue, 2 Feb 2021 23:17:03 +0200 Subject: [PATCH 81/81] always use UTF-8 when parsing XML --- src/test_xml_parser/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_xml_parser/core.clj b/src/test_xml_parser/core.clj index 26f0136..e2d778d 100644 --- a/src/test_xml_parser/core.clj +++ b/src/test_xml_parser/core.clj @@ -17,7 +17,7 @@ (defn zip-str [s] (zip/xml-zip - (xml/parse (java.io.ByteArrayInputStream. (.getBytes s))))) + (xml/parse (java.io.ByteArrayInputStream. (.getBytes s "UTF-8"))))) (defn filter-tags [xml-content tag-key] (let [filter-result (filter #(= (:tag %) tag-key) xml-content)]