From 54e71ab01d0e9e2bee410037f2c161e2463cbbb7 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 27 Dec 2016 14:31:34 +0200 Subject: [PATCH 1/9] Disable unused Statistics --- data/dataIO.js | 21 +-------------------- package.json | 3 +-- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/data/dataIO.js b/data/dataIO.js index 0523404..d1320ca 100644 --- a/data/dataIO.js +++ b/data/dataIO.js @@ -15,19 +15,6 @@ const giveConfig = require('../src.js').config const dataIO = {} module.exports = dataIO -const url = require('url') -const parsedStatsdUrl = url.parse(process.env.STATSD_URL, true, true) -const StatsD = require('node-statsd') -const statsClient = new StatsD( - { - host: parsedStatsdUrl.hostname, - port: parsedStatsdUrl.port - } - ) -statsClient.socket.on('error', function (error) { - _log_(`Error in socket: ${error}`) -}) - const firstEvent = 0 /** @@ -166,8 +153,6 @@ dataIO.sendtoAPI = (title, agenda, social, place, regUrl, imgUrl, whenStart, whe 'submitter_email': process.env.EMAIL }) - statsClient.increment('eventsparser.sendToAPI.sends') - const options = { 'hostname': process.env.HOSTNAME_URL, 'port': process.env.HOSTNAME_PORT, @@ -187,11 +172,7 @@ dataIO.sendtoAPI = (title, agenda, social, place, regUrl, imgUrl, whenStart, whe res.on('end', () => { _log_('No more data in response. \n') }) } }) - req.on('error', (e) => { - _log_(`problem with request: ${e.message}`) - statsClient.increment('eventsparser.sendToAPI.errors') - }) - + req.on('error', (e) => { _log_(`problem with request: ${e.message}`) }) req.write(body) req.end() } diff --git a/package.json b/package.json index ce21e47..4730888 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,7 @@ "moment": "~2.13.0", "node-statsd": "0.1.1", "sync-request": "~3.0.0", - "url": "0.11.0", - "xml2json": "0.9.2", + "xml2json": "~0.9.2", "yandex-translate": "~2.1.0" }, "devDependencies": { From a10d98c7cc0a1031442fdc30655b06dc6b65ced1 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 27 Dec 2016 14:36:07 +0200 Subject: [PATCH 2/9] Revert other unused code for statistic --- data/dataIO.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data/dataIO.js b/data/dataIO.js index d1320ca..0d141c3 100644 --- a/data/dataIO.js +++ b/data/dataIO.js @@ -34,8 +34,6 @@ dataIO.get = (srcName, srcType, srcLink, newJSON, oldJSON) => { // Save old data fs.copySync(newJSON, oldJSON) - statsClient.increment('eventsparser.get') - // Get data from source const res = request('GET', srcLink).getBody() @@ -78,8 +76,6 @@ dataIO.read = (srcFrom, file) => { let data = fs.readJsonSync(file, {'throws': false}) data = eval(giveConfig[srcFrom].allEvents) - statsClient.gauge(`eventsparser.sources.${srcFrom}.count`, data.length) - return data } From 28e5286c770e43cb86c717588c40ff4dcba8d911 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Tue, 27 Dec 2016 16:07:36 +0200 Subject: [PATCH 3/9] Fix vagga config crash: add git --- vagga.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vagga.yaml b/vagga.yaml index cae51e6..e99dd69 100644 --- a/vagga.yaml +++ b/vagga.yaml @@ -24,7 +24,7 @@ containers: # Python is a dendency of node-gyp: # Ref: https://github.com/nodejs/node-gyp#installation - !BuildDeps [python] - - !Install [libexpat1] + - !Install [libexpat1, git] # install dependencies from package.json, see the doc for extra options: # Ref: vagga.readthedocs.io/en/latest/build_steps.html#step-NpmDependencies - !NpmDependencies @@ -92,7 +92,7 @@ containers: eventsparser: setup: - !Alpine v3.4 - - !BuildDeps [python, git] + - !Install [python, git] - !NpmDependencies file: package.json environ: From 2e1d3ad4a50befa47c55438b81075ab496ee101f Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Wed, 28 Dec 2016 21:12:16 +0200 Subject: [PATCH 4/9] Add Docker --- Dockerfile | 10 ++++++++++ README.md | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a57b144 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM node:latest +MAINTAINER Maksym Vlasov +LABEL Name=events-parser Version=0.5.1 +COPY package.json /tmp/package.json +RUN cd /tmp && npm install --production +RUN mkdir -p /usr/src/app && mv /tmp/node_modules /usr/src +WORKDIR /usr/src/app +COPY . /usr/src/app +EXPOSE 3000 +CMD node main.js diff --git a/README.md b/README.md index c8f4f9b..5a336f0 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Licensed under the Apache License, Version 2.0. + * [Vagga](http://vagga.readthedocs.io/en/latest/installation.html) (higly recomended) ++ +* [Docker](https://docs.docker.com/engine/getstarted/)
OR * [Node.JS](https://nodejs.org/uk/) 4+ (not recomended) @@ -65,6 +67,29 @@ $ vagga run ``` +### Docker-way + + +#### Add enviroment keys +Add variables to your `Dockerfile` on this pattern: +```bash +ENV %variable% %value% +``` + +**[All supported enviroment variables](#all-supported-enviroment-variables)** + +#### Build image +```bash +$ docker build -f Dockerfile -t events-parser:latest . +``` + +#### Run +```bash +$ docker run --rm events-parser:latest +``` + + + ### NodeJS-way From 6e349764a7df5ab6ab81c755f25677428399307c Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Wed, 28 Dec 2016 21:24:10 +0200 Subject: [PATCH 5/9] Fdd docker-travis build --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.travis.yml b/.travis.yml index 80d6ced..01ed917 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,3 +33,19 @@ install: vagga _build nodejs-7-ubuntu-bin # Run tests script: vagga test + +# Docker analog +sudo: required + +language: javascript + +services: + - docker + +before_install: + - docker build -t events-parser:latest . + - docker run --rm events-parser:latest + - docker ps -a + +script: + - standard \ No newline at end of file From d226c0e7fef3f3f8cb5d18c339b7e65e04d666b2 Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Wed, 28 Dec 2016 21:32:12 +0200 Subject: [PATCH 6/9] Fix docker-travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 01ed917..4160313 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,4 +48,5 @@ before_install: - docker ps -a script: + - npm i standard - standard \ No newline at end of file From f8a6c790fec6661db406d706ca0d2f0a3a633abb Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 29 Dec 2016 01:58:23 +0200 Subject: [PATCH 7/9] Revert "Merge branch 'vagga' into tests" This reverts commit de78e0ff3364e4a79ce9de299d23bb79d379a342, reversing changes made to a10d98c7cc0a1031442fdc30655b06dc6b65ced1. --- vagga.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vagga.yaml b/vagga.yaml index e99dd69..cae51e6 100644 --- a/vagga.yaml +++ b/vagga.yaml @@ -24,7 +24,7 @@ containers: # Python is a dendency of node-gyp: # Ref: https://github.com/nodejs/node-gyp#installation - !BuildDeps [python] - - !Install [libexpat1, git] + - !Install [libexpat1] # install dependencies from package.json, see the doc for extra options: # Ref: vagga.readthedocs.io/en/latest/build_steps.html#step-NpmDependencies - !NpmDependencies @@ -92,7 +92,7 @@ containers: eventsparser: setup: - !Alpine v3.4 - - !Install [python, git] + - !BuildDeps [python, git] - !NpmDependencies file: package.json environ: From 0a042336bcb1fe4c891f8ebfa9f0c6a502f58b6a Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 29 Dec 2016 01:58:32 +0200 Subject: [PATCH 8/9] Revert "Revert other unused code for statistic" This reverts commit a10d98c7cc0a1031442fdc30655b06dc6b65ced1. --- data/dataIO.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/dataIO.js b/data/dataIO.js index 0d141c3..d1320ca 100644 --- a/data/dataIO.js +++ b/data/dataIO.js @@ -34,6 +34,8 @@ dataIO.get = (srcName, srcType, srcLink, newJSON, oldJSON) => { // Save old data fs.copySync(newJSON, oldJSON) + statsClient.increment('eventsparser.get') + // Get data from source const res = request('GET', srcLink).getBody() @@ -76,6 +78,8 @@ dataIO.read = (srcFrom, file) => { let data = fs.readJsonSync(file, {'throws': false}) data = eval(giveConfig[srcFrom].allEvents) + statsClient.gauge(`eventsparser.sources.${srcFrom}.count`, data.length) + return data } From b619f5dedf81c71266c33c0b76e6040c77ecc79d Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Thu, 29 Dec 2016 01:58:48 +0200 Subject: [PATCH 9/9] Revert "Disable unused Statistics" This reverts commit 54e71ab01d0e9e2bee410037f2c161e2463cbbb7. --- data/dataIO.js | 21 ++++++++++++++++++++- package.json | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/data/dataIO.js b/data/dataIO.js index d1320ca..0523404 100644 --- a/data/dataIO.js +++ b/data/dataIO.js @@ -15,6 +15,19 @@ const giveConfig = require('../src.js').config const dataIO = {} module.exports = dataIO +const url = require('url') +const parsedStatsdUrl = url.parse(process.env.STATSD_URL, true, true) +const StatsD = require('node-statsd') +const statsClient = new StatsD( + { + host: parsedStatsdUrl.hostname, + port: parsedStatsdUrl.port + } + ) +statsClient.socket.on('error', function (error) { + _log_(`Error in socket: ${error}`) +}) + const firstEvent = 0 /** @@ -153,6 +166,8 @@ dataIO.sendtoAPI = (title, agenda, social, place, regUrl, imgUrl, whenStart, whe 'submitter_email': process.env.EMAIL }) + statsClient.increment('eventsparser.sendToAPI.sends') + const options = { 'hostname': process.env.HOSTNAME_URL, 'port': process.env.HOSTNAME_PORT, @@ -172,7 +187,11 @@ dataIO.sendtoAPI = (title, agenda, social, place, regUrl, imgUrl, whenStart, whe res.on('end', () => { _log_('No more data in response. \n') }) } }) - req.on('error', (e) => { _log_(`problem with request: ${e.message}`) }) + req.on('error', (e) => { + _log_(`problem with request: ${e.message}`) + statsClient.increment('eventsparser.sendToAPI.errors') + }) + req.write(body) req.end() } diff --git a/package.json b/package.json index 4730888..ce21e47 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "moment": "~2.13.0", "node-statsd": "0.1.1", "sync-request": "~3.0.0", - "xml2json": "~0.9.2", + "url": "0.11.0", + "xml2json": "0.9.2", "yandex-translate": "~2.1.0" }, "devDependencies": {