diff --git a/eddbimporter/commodities.go b/eddbimporter/commodities.go new file mode 100644 index 0000000..68568d8 --- /dev/null +++ b/eddbimporter/commodities.go @@ -0,0 +1,191 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func ProcessCommodities(client *mongo.Client) bool { + return false +} + +/* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const path = require('path'); +// const fs = require('fs-extra'); +// const commoditiesModel = require('../../models/commodities'); +// const utilities = require('../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// let fileSize = require('../utilities/file_size'); + +// module.exports = Commodities; + +// const pathToFile = path.resolve(__dirname, '../../dumps/listings.csv'); + +// function Commodities() { +// eventEmmiter.call(this); + +// this.update = function () { +// let recordsUpdated = 0; +// new utilities.csvToJson(pathToFile) +// .on('start', () => { +// console.log(`EDDB commodity dump update reported`); +// this.emit('started', { +// statusCode: 200, +// update: "started", +// type: 'commodity' +// }); +// }) +// .on('json', json => { +// commoditiesModel +// .then(model => { +// model.findOneAndUpdate( +// { id: json.id }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// fs.unlink(pathToFile, () => { +// console.log('Commodity Dump deleted'); +// }); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.import = function () { +// let recordsInserted = 0; +// new utilities.csvToJson(pathToFile) +// .on('start', () => { +// console.log(`EDDB commodity dump insertion reported`); +// this.emit('started', { +// statusCode: 200, +// insertion: "started", +// type: 'commodity' +// }); +// }) +// .on('json', json => { +// commoditiesModel +// .then(model => { +// let document = new model(json); +// document.save() +// .then(() => { +// recordsInserted++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsInserted} records inserted`); +// fs.unlink(pathToFile, () => { +// console.log('Commodity Dump deleted'); +// }); +// this.emit('done', recordsInserted); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.download = function () { +// new utilities.download('https://eddb.io/archive/v5/listings.csv', pathToFile) +// .on('start', response => { +// console.log(`EDDB commodity dump reported with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'commodity' +// }); +// }) +// .on('end', () => { +// console.log(`EDDB commodity dump saved successfully with file size ${fileSize.withPath(pathToFile)}`) +// this.emit('done'); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v5/listings.csv', 'csv') +// .on('start', response => { +// console.log(`EDDB commodity dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'commodity' +// }); +// }) +// .on('json', json => { +// commoditiesModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(Commodities, eventEmmiter); diff --git a/eddbimporter/commoditiesv6.go b/eddbimporter/commoditiesv6.go new file mode 100644 index 0000000..8006de1 --- /dev/null +++ b/eddbimporter/commoditiesv6.go @@ -0,0 +1,83 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func CheckAndInsertCommoditiesV6Schema(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const commoditiesModel = require('../../../models/v6/commodities'); +// const utilities = require('../../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// module.exports = Commodities; + +// function Commodities() { +// eventEmmiter.call(this); + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v6/listings.csv', 'csv') +// .on('start', response => { +// console.log(`EDDB commodity dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'commodity' +// }); +// }) +// .on('json', json => { +// commoditiesModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(Commodities, eventEmmiter); diff --git a/eddbimporter/config.go b/eddbimporter/config.go new file mode 100644 index 0000000..aa27911 --- /dev/null +++ b/eddbimporter/config.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "os" + + "github.com/ilyakaznacheev/cleanenv" +) + +const configFile = "config.yml" + +// ConfigDatabase provides default values, which can be overriden by the configuration file and environment variables +type ConfigDatabase struct { + Collection string `yaml:"port" env:"PORT" env-default:"eddb"` + Port string `yaml:"port" env:"PORT" env-default:"27017"` + Host string `yaml:"host" env:"HOST" env-default:"localhost"` + User string `yaml:"user" env:"USER" env-default:""` + Password string `yaml:"password" env:""` +} + +// Cfg is a global variable containing the configuration details +var Cfg ConfigDatabase + +// ReadConfig() reads the configuration using the defaults +// The defaults can be overridden in the YAML file config.yml +// or lastly from environment variables +func ReadConfig() { + fmt.Print("Reading configuration... ") + + err := cleanenv.ReadConfig(configFile, &cfg) + if err != nil { + fmt.Println(" could not read the configuration") + os.Exit(1) + } + + fmt.Println("ok") +} diff --git a/eddbimporter/config.yml b/eddbimporter/config.yml new file mode 100755 index 0000000..4ebad24 --- /dev/null +++ b/eddbimporter/config.yml @@ -0,0 +1,7 @@ +collection: "eddb" +port: 27017 +host: "localhost" +user: "" +password: "" +https://eddb.io/archive/v6/bodies_recently.jsonl + diff --git a/eddbimporter/db.go b/eddbimporter/db.go new file mode 100644 index 0000000..c0a5783 --- /dev/null +++ b/eddbimporter/db.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// DbClient singleton representing the database connection +// Go routines will need to clone this to make their own independant connection to the database +var DbClient *mongo.Client + +// Connect to Mongo DB +func connectMongo() bool { + + fmt.Print("Connecting to MongoDB... ") + connStr := "mongodb://" + cfg.User + ":" + cfg.Password + "@" + cfg.Host + ":" + cfg.Port + DbClient, err := mongo.NewClient(options.Client().ApplyURI(connStr)) + if err != nil { + fmt.Print(" could not create MongoDB client") + os.Exit(1) + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + err = DbClient.Connect(ctx) + if err != nil { + fmt.Print(" could not connect to database") + os.Exit(1) + } + + fmt.Println("ok") + + return true +} diff --git a/eddbimporter/factions.go b/eddbimporter/factions.go new file mode 100644 index 0000000..01de562 --- /dev/null +++ b/eddbimporter/factions.go @@ -0,0 +1,192 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func ProcessFactions(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const path = require('path'); +// const fs = require('fs-extra'); +// const factionsModel = require('../../models/factions'); +// const utilities = require('../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// let fileSize = require('../utilities/file_size'); + +// module.exports = Factions; + +// const pathToFile = path.resolve(__dirname, '../../dumps/factions.json'); + +// function Factions() { +// eventEmmiter.call(this); + +// this.update = function () { +// let recordsUpdated = 0; +// new utilities.jsonParse(pathToFile) +// .on('start', () => { +// console.log(`EDDB faction dump update reported`); +// this.emit('started', { +// statusCode: 200, +// update: "started", +// type: 'faction' +// }); +// }) +// .on('json', json => { +// factionsModel +// .then(model => { +// model.findOneAndUpdate( +// { id: json.id }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// fs.unlink(pathToFile, () => { +// console.log('Faction Dump deleted'); +// }); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.import = function () { +// let recordsInserted = 0; +// new utilities.jsonParse(pathToFile) +// .on('start', () => { +// console.log(`EDDB faction dump insertion reported`); +// this.emit('started', { +// statusCode: 200, +// insertion: "started", +// type: 'faction' +// }); +// }) +// .on('json', json => { +// factionsModel +// .then(model => { +// let document = new model(json); +// document.save() +// .then(() => { +// recordsInserted++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsInserted} records inserted`); +// fs.unlink(pathToFile, () => { +// console.log('Faction Dump deleted'); +// }); +// this.emit('done', recordsInserted); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.download = function () { +// new utilities.download('https://eddb.io/archive/v5/factions.json', pathToFile) +// .on('start', response => { +// console.log(`EDDB faction dump reported with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'faction' +// }); +// }) +// .on('end', () => { +// console.log(`EDDB faction dump saved successfully with file size ${fileSize.withPath(pathToFile)}`) +// this.emit('done'); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v5/factions.json', 'json') +// .on('start', response => { +// console.log(`EDDB faction dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'faction' +// }); +// }) +// .on('json', json => { +// factionsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(Factions, eventEmmiter); diff --git a/eddbimporter/factionsv6.go b/eddbimporter/factionsv6.go new file mode 100644 index 0000000..4a16091 --- /dev/null +++ b/eddbimporter/factionsv6.go @@ -0,0 +1,84 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func CheckAndInsertFactionV6Schema(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const factionsModel = require('../../../models/v6/factions'); +// const utilities = require('../../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// module.exports = Factions; + +// function Factions() { +// eventEmmiter.call(this); + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v6/factions.json', 'json') +// .on('start', response => { +// console.log(`EDDB faction dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'faction' +// }); +// }) +// .on('json', json => { +// factionsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(Factions, eventEmmiter); diff --git a/eddbimporter/go.mod b/eddbimporter/go.mod new file mode 100644 index 0000000..f35cf38 --- /dev/null +++ b/eddbimporter/go.mod @@ -0,0 +1,8 @@ +module kodeblox.com/eddbapi + +go 1.15 + +require ( + github.com/ilyakaznacheev/cleanenv v1.2.5 + go.mongodb.org/mongo-driver v1.4.6 +) diff --git a/eddbimporter/go.sum b/eddbimporter/go.sum new file mode 100644 index 0000000..35156ac --- /dev/null +++ b/eddbimporter/go.sum @@ -0,0 +1,120 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/ilyakaznacheev/cleanenv v1.2.5 h1:/SlcF9GaIvefWqFJzsccGG/NJdoaAwb7Mm7ImzhO3DM= +github.com/ilyakaznacheev/cleanenv v1.2.5/go.mod h1:/i3yhzwZ3s7hacNERGFwvlhwXMDcaqwIzmayEhbRplk= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +go.mongodb.org/mongo-driver v1.4.6 h1:rh7GdYmDrb8AQSkF8yteAus8qYOgOASWDOv1BWqBXkU= +go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5 h1:8dUaAV7K4uHsF56JQWkprecIQKdPHtR9jCHF5nB8uzc= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +olympos.io/encoding/edn v0.0.0-20200308123125-93e3b8dd0e24 h1:sreVOrDp0/ezb0CHKVek/l7YwpxPJqv+jT3izfSphA4= +olympos.io/encoding/edn v0.0.0-20200308123125-93e3b8dd0e24/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= diff --git a/eddbimporter/main.go b/eddbimporter/main.go new file mode 100755 index 0000000..c7ff286 --- /dev/null +++ b/eddbimporter/main.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/ilyakaznacheev/cleanenv" +) + + + +func insertCommodities(bodies chan *mongo.Client) { + // Wait for the MongoDB client to be passed and then get going + var client *mongo.Client + client <- bodies + + // ensure that the collection has the schema, and + CheckAndInsertCommoditiesV6Schema(client) + + // download and process the collection + ProcessCommodities(client) +} + +func main() { + + fmt.Println("EliteBGS EDDB API Importer " + Eddb_importer_api + " starting up") + + // read the configuration from defaults > config.yml > environment variables into global cfg + readConfig() + + // Connect to MongoDB + client := connectMongo() + + startTime := time.Now() + + commodities := make(chan *mongo.Client, 1) + go insertCommodities(commodities) + commodities <- client + + // insertCommodities(client) + // insertFactions(client) + // insertSystems(client) + // insertStations(client) + // insertPopulatedSystems(client) + + <-commodities + + endTime := time.Now() + + fmt.Println("Import completed. Time taken: ", endTime.Sub(startTime)) +} diff --git a/eddbimporter/populated_systems.go b/eddbimporter/populated_systems.go new file mode 100644 index 0000000..74911b1 --- /dev/null +++ b/eddbimporter/populated_systems.go @@ -0,0 +1,192 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func ProcessPopulatedSystems(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const path = require('path'); +// const fs = require('fs-extra'); +// const populatedSystemsModel = require('../../models/populated_systems'); +// const utilities = require('../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// let fileSize = require('../utilities/file_size'); + +// module.exports = PopulatedSystems; + +// const pathToFile = path.resolve(__dirname, '../../dumps/systems_populated.json'); + +// function PopulatedSystems() { +// eventEmmiter.call(this); + +// this.update = function () { +// let recordsUpdated = 0; +// new utilities.jsonParse(pathToFile) +// .on('start', () => { +// console.log(`EDDB populated system dump update reported`); +// this.emit('started', { +// statusCode: 200, +// update: "started", +// type: 'populated system' +// }); +// }) +// .on('json', json => { +// populatedSystemsModel +// .then(model => { +// model.findOneAndUpdate( +// { id: json.id }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// fs.unlink(pathToFile, () => { +// console.log('Populated System Dump deleted'); +// }); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.import = function () { +// let recordsInserted = 0; +// new utilities.jsonParse(pathToFile) +// .on('start', () => { +// console.log(`EDDB populated system dump insertion reported`); +// this.emit('started', { +// statusCode: 200, +// insertion: "started", +// type: 'populated system' +// }); +// }) +// .on('json', json => { +// populatedSystemsModel +// .then(model => { +// let document = new model(json); +// document.save() +// .then(() => { +// recordsInserted++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsInserted} records inserted`); +// fs.unlink(pathToFile, () => { +// console.log('Populated System Dump deleted'); +// }); +// this.emit('done', recordsInserted); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.download = function () { +// new utilities.download('https://eddb.io/archive/v5/systems_populated.json', pathToFile) +// .on('start', response => { +// console.log(`EDDB populated system dump reported with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'populated system' +// }); +// }) +// .on('end', () => { +// console.log(`EDDB populated system dump saved successfully with file size ${fileSize.withPath(pathToFile)}`) +// this.emit('done'); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v5/systems_populated.json', 'json') +// .on('start', response => { +// console.log(`EDDB populated system dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'populated system' +// }); +// }) +// .on('json', json => { +// populatedSystemsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(PopulatedSystems, eventEmmiter); diff --git a/eddbimporter/populated_systemsv6.go b/eddbimporter/populated_systemsv6.go new file mode 100644 index 0000000..d32a019 --- /dev/null +++ b/eddbimporter/populated_systemsv6.go @@ -0,0 +1,103 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func CheckAndInsertPopulatedSystemsV6Schema(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const populatedSystemsModel = require('../../../models/v6/populated_systems'); +// const utilities = require('../../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// module.exports = PopulatedSystems; + +// function PopulatedSystems() { +// eventEmmiter.call(this); + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v6/systems_populated.json', 'json') +// .on('start', response => { +// console.log(`EDDB populated system dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'populated system' +// }); +// }) +// .on('json', json => { +// json.states = statify(json.states); +// json.minor_faction_presences.forEach((minor_faction_presence, index, minor_faction_presences) => { +// minor_faction_presences[index].active_states = statify(minor_faction_presence.active_states); +// minor_faction_presences[index].pending_states = statify(minor_faction_presence.pending_states); +// minor_faction_presences[index].recovering_states = statify(minor_faction_presence.recovering_states); +// }); +// populatedSystemsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } + +// let statify = ref => { +// let entities = ref; +// ref = []; +// entities.forEach((entity, index, allEntities) => { +// ref.push({ +// id: entity.id, +// name: entity.name, +// name_lower: entity.name.toLowerCase() +// }); +// }, this); +// return ref; +// } +// } + +// inherits(PopulatedSystems, eventEmmiter); diff --git a/eddbimporter/stations.go b/eddbimporter/stations.go new file mode 100644 index 0000000..fa72f61 --- /dev/null +++ b/eddbimporter/stations.go @@ -0,0 +1,219 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func ProcessStations(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const path = require('path'); +// const fs = require('fs-extra'); +// const stationsModel = require('../../models/stations'); +// const utilities = require('../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// let fileSize = require('../utilities/file_size'); + +// module.exports = Stations; + +// const pathToFile = path.resolve(__dirname, '../../dumps/stations.json'); + +// function Stations() { +// eventEmmiter.call(this); + +// this.update = function () { +// let recordsUpdated = 0; +// new utilities.jsonParse(pathToFile) +// .on('start', () => { +// console.log(`EDDB station dump update reported`); +// this.emit('started', { +// statusCode: 200, +// update: "started", +// type: 'station' +// }); +// }) +// .on('json', json => { +// json.import_commodities = objectify(json.import_commodities); +// json.export_commodities = objectify(json.export_commodities); +// json.prohibited_commodities = objectify(json.prohibited_commodities); +// json.economies = objectify(json.economies); +// json.selling_ships = objectify(json.selling_ships); +// stationsModel +// .then(model => { +// model.findOneAndUpdate( +// { id: json.id }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// fs.unlink(pathToFile, () => { +// console.log('Station Dump deleted'); +// }); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.import = function () { +// let recordsInserted = 0; +// new utilities.jsonParse(pathToFile) +// .on('start', () => { +// console.log(`EDDB station dump insertion reported`); +// this.emit('started', { +// statusCode: 200, +// insertion: "started", +// type: 'station' +// }); +// }) +// .on('json', json => { +// json.import_commodities = objectify(json.import_commodities); +// json.export_commodities = objectify(json.export_commodities); +// json.prohibited_commodities = objectify(json.prohibited_commodities); +// json.economies = objectify(json.economies); +// json.selling_ships = objectify(json.selling_ships); +// stationsModel +// .then(model => { +// let document = new model(json); +// document.save() +// .then(() => { +// recordsInserted++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsInserted} records inserted`); +// fs.unlink(pathToFile, () => { +// console.log('Station Dump deleted'); +// }); +// this.emit('done', recordsInserted); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.download = function () { +// new utilities.download('https://eddb.io/archive/v5/stations.json', pathToFile) +// .on('start', response => { +// console.log(`EDDB station reported with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'station' +// }); +// }) +// .on('end', () => { +// console.log(`EDDB station dump saved successfully with file size ${fileSize.withPath(pathToFile)}`) +// this.emit('done'); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v5/stations.json', 'json') +// .on('start', response => { +// console.log(`EDDB station dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'station' +// }); +// }) +// .on('json', json => { +// json.import_commodities = objectify(json.import_commodities); +// json.export_commodities = objectify(json.export_commodities); +// json.prohibited_commodities = objectify(json.prohibited_commodities); +// json.economies = objectify(json.economies); +// json.selling_ships = objectify(json.selling_ships); +// stationsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } + +// let objectify = ref => { +// let entities = ref; +// ref = []; +// entities.forEach((entity, index, allEntities) => { +// ref.push({ +// name: entity, +// name_lower: entity.toLowerCase() +// }); +// }, this); +// return ref; +// } +// } + +// inherits(Stations, eventEmmiter); diff --git a/eddbimporter/stationsv6.go b/eddbimporter/stationsv6.go new file mode 100644 index 0000000..3c7fe24 --- /dev/null +++ b/eddbimporter/stationsv6.go @@ -0,0 +1,115 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func CheckAndInsertStationsV6Schema(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const stationsModel = require('../../../models/v6/stations'); +// const utilities = require('../../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// module.exports = Stations; + +// function Stations() { +// eventEmmiter.call(this); + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v6/stations.json', 'json') +// .on('start', response => { +// console.log(`EDDB station dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'station' +// }); +// }) +// .on('json', json => { +// json.states = statify(json.states); +// json.import_commodities = objectify(json.import_commodities); +// json.export_commodities = objectify(json.export_commodities); +// json.prohibited_commodities = objectify(json.prohibited_commodities); +// json.economies = objectify(json.economies); +// json.selling_ships = objectify(json.selling_ships); +// stationsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } + +// let objectify = ref => { +// let entities = ref; +// ref = []; +// entities.forEach((entity, index, allEntities) => { +// ref.push({ +// name: entity, +// name_lower: entity.toLowerCase() +// }); +// }, this); +// return ref; +// } + +// let statify = ref => { +// let entities = ref; +// ref = []; +// entities.forEach((entity, index, allEntities) => { +// ref.push({ +// id: entity.id, +// name: entity.name, +// name_lower: entity.name.toLowerCase() +// }); +// }, this); +// return ref; +// } +// } + +// inherits(Stations, eventEmmiter); diff --git a/eddbimporter/systems.go b/eddbimporter/systems.go new file mode 100644 index 0000000..5f28c2b --- /dev/null +++ b/eddbimporter/systems.go @@ -0,0 +1,192 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func ProcessSystems(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const path = require('path'); +// const fs = require('fs-extra'); +// const systemsModel = require('../../models/systems'); +// const utilities = require('../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// let fileSize = require('../utilities/file_size'); + +// module.exports = Systems; + +// const pathToFile = path.resolve(__dirname, '../../dumps/systems.csv'); + +// function Systems() { +// eventEmmiter.call(this); + +// this.update = function () { +// let recordsUpdated = 0; +// new utilities.csvToJson(pathToFile) +// .on('start', () => { +// console.log(`EDDB system dump update reported`); +// this.emit('started', { +// statusCode: 200, +// update: "started", +// type: 'system' +// }); +// }) +// .on('json', json => { +// systemsModel +// .then(model => { +// model.findOneAndUpdate( +// { id: json.id }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// fs.unlink(pathToFile, () => { +// console.log('System Dump deleted'); +// }); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.import = function () { +// let recordsInserted = 0; +// new utilities.csvToJson(pathToFile) +// .on('start', () => { +// console.log(`EDDB system dump insertion reported`); +// this.emit('started', { +// statusCode: 200, +// insertion: "started", +// type: 'system' +// }); +// }) +// .on('json', json => { +// systemsModel +// .then(model => { +// let document = new model(json); +// document.save() +// .then(() => { +// recordsInserted++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsInserted} records inserted`); +// fs.unlink(pathToFile, () => { +// console.log('System Dump deleted'); +// }); +// this.emit('done', recordsInserted); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// }; + +// this.download = function () { +// new utilities.download('https://eddb.io/archive/v5/systems.csv', pathToFile) +// .on('start', response => { +// console.log(`EDDB system dump reported with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'system' +// }); +// }) +// .on('end', () => { +// console.log(`EDDB system dump saved successfully with file size ${fileSize.withPath(pathToFile)}`) +// this.emit('done'); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v5/systems.csv', 'csv') +// .on('start', response => { +// console.log(`EDDB system dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'station' +// }); +// }) +// .on('json', json => { +// systemsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(Systems, eventEmmiter); diff --git a/eddbimporter/systemsv6.go b/eddbimporter/systemsv6.go new file mode 100644 index 0000000..fe9fdbb --- /dev/null +++ b/eddbimporter/systemsv6.go @@ -0,0 +1,84 @@ +package main + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func CheckAndInsertSystemsV6Schema(client *mongo.Client) bool { + return false +} + +// /* +// * KodeBlox Copyright 2018 Sayak Mukhopadhyay +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http: //www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +// "use strict"; + +// const systemsModel = require('../../../models/v6/systems'); +// const utilities = require('../../utilities'); +// const eventEmmiter = require('events').EventEmitter; +// const inherits = require('util').inherits; + +// module.exports = Systems; + +// function Systems() { +// eventEmmiter.call(this); + +// this.downloadUpdate = function () { +// let recordsUpdated = 0; +// new utilities.downloadUpdate('https://eddb.io/archive/v6/systems.csv', 'csv') +// .on('start', response => { +// console.log(`EDDB system dump started with status code ${response.statusCode}`); +// this.emit('started', { +// response: response, +// insertion: "started", +// type: 'station' +// }); +// }) +// .on('json', json => { +// systemsModel +// .then(model => { +// model.findOneAndUpdate( +// { +// id: json.id, +// updated_at: { $ne: json.updated_at } +// }, +// json, +// { +// upsert: true, +// runValidators: true +// }) +// .then(() => { +// recordsUpdated++; +// }) +// .catch((err) => { +// this.emit('error', err); +// }); +// }) +// .catch(err => { +// this.emit('error', err); +// }); +// }) +// .on('end', () => { +// console.log(`${recordsUpdated} records updated`); +// this.emit('done', recordsUpdated); +// }) +// .on('error', err => { +// this.emit('error', err); +// }) +// } +// } + +// inherits(Systems, eventEmmiter); diff --git a/eddbimporter/utlities.go b/eddbimporter/utlities.go new file mode 100644 index 0000000..de2d72c --- /dev/null +++ b/eddbimporter/utlities.go @@ -0,0 +1,184 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "time" +) + +// GetJSON - obtain the JSON from a remote URL +func GetJSON(url string, target interface{}) error { + httpClient := &http.Client{Timeout: 10 * time.Second} + + res, err := httpClient.Get(url) + if err != nil { + fmt.Println("Cannot obtain data from ", url) + os.Exit(1) + } + defer res.Body.Close() + + return json.NewDecoder(res.Body).Decode(target) +} + +// CsvToJSON converts a CSV file stored at path to a JSON string +func CsvToJSON(path string) string { + // function CsvToJson(path) { + // eventEmmiter.call(this); + // let firstData = true; + // csvtojson() + // .fromStream(fs.createReadStream(path)) + // .on('json', json => { + // if (firstData) { + // firstData = !firstData; + // this.emit('start'); + // } + // this.emit('json', json); + // }) + // .on('done', (error) => { + // if (error) { + // this.emit('error', error); + // } else { + // this.emit('end'); + // } + // }); + // } + return "[]" +} + +// DownloadUpdate downloads an update from the eddb website +func DownloadUpdate(pathFrom string, filetype string) { + // function DownloadUpdate(pathFrom, type) { + // eventEmmiter.call(this); + // if (type === 'jsonl') { + // request.get(pathFrom, { headers: { 'Accept-Encoding': 'gzip, deflate, sdch' }, gzip: true }) + // .on('response', response => { + // response.statusCode = 200; + // this.emit('start', response); + // }) + // .pipe(ndjson.parse()) + // .on('data', json => { + // this.emit('json', json); + // }) + // .on('end', () => { + // this.emit('end'); + // }) + // .on('error', error => { + // this.emit('error', error); + // }) + // } else if (type === 'json') { + // request.get(pathFrom, { headers: { 'Accept-Encoding': 'gzip, deflate, sdch' }, gzip: true }) + // .on('response', response => { + // response.statusCode = 200; + // this.emit('start', response); + // }) + // .pipe(jsonStream.parse('*')) + // .on('data', json => { + // this.emit('json', json); + // }) + // .on('end', () => { + // this.emit('end'); + // }) + // .on('error', error => { + // this.emit('error', error); + // }) + // } else if (type === 'csv') { + // csvtojson() + // .fromStream(request.get(pathFrom, { headers: { 'Accept-Encoding': 'gzip, deflate, sdch' }, gzip: true }) + // .on('response', response => { + // response.statusCode = 200; + // this.emit('start', response); + // })) + // .on('json', json => { + // this.emit('json', json); + // }) + // .on('done', (error) => { + // if (error) { + // this.emit('error', error); + // } else { + // this.emit('end'); + // } + // }); + // } + // } +} + +// Download - download a full dump from eddb and puts it into a file at pathTo +func Download(pathFrom string, pathTo string) { + // function Download(pathFrom, pathTo) { + // eventEmmiter.call(this); + // let progressPercent = 0.0; + // progress(request.get(pathFrom, {headers: {'Accept-Encoding': 'gzip, deflate, sdch'}, gzip: true})) + // .on('response', response => { + // response.statusCode = 200; + // this.emit('start', response); + // }) + // .on('progress', status => { + // if (process.env.NODE_ENV === 'development') { + // console.log(`Downloading File of size ${fileSize.withValue(status.size.total)}\nTime Elapsed: ${status.time.elapsed}\t Time Remaining: ${status.time.remaining}\n${(status.percent * 100).toFixed(2)}% completed\t ${fileSize.withValue(status.size.transferred)} data transferred`); + // } + // progressPercent = status.percent * 100; + // }) + // .on('error', err => { + // this.emit('error', { + // error: err, + // progress: progressPercent + // }); + // }) + // .pipe(fs.createWriteStream(pathTo) + // .on('finish', () => { + // this.emit('end'); + // }) + // .on('error', err => { + // this.emit('error', { + // error: err, + // progress: progressPercent + // }); + // })); + // } +} + +func JsonParse(path string) { + // function JsonParse(path) { + // eventEmmiter.call(this); + // let firstData = true; + // fs.createReadStream(path) + // .pipe(jsonStream.parse('*')) + // .on('data', json => { + // if (firstData) { + // firstData = false; + // this.emit('start'); + // } + // this.emit('json', json); + // }) + // .on('end', () => { + // this.emit('end'); + // }) + // .on('error', error => { + // this.emit('error', error); + // }); + // } +} + +func JsonlToJson(path string) { + // function JsonlToJson(path) { + // eventEmmiter.call(this); + // let firstData = true; + // fs.createReadStream(path) + // .pipe(ndjson.parse()) + // .on('data', json => { + // if (firstData) { + // firstData = !firstData; + // this.emit('start'); + // } + // this.emit('json', json); + // }) + // .on('end', () => { + // this.emit('end'); + // }) + // .on('error', error => { + // this.emit('error', error); + // }) + // } +} diff --git a/eddbimporter/version.go b/eddbimporter/version.go new file mode 100644 index 0000000..7208b60 --- /dev/null +++ b/eddbimporter/version.go @@ -0,0 +1,3 @@ +package main + +const Eddb_importer_api = "v2.1.0"