IEEE.org     |     IEEE Xplore Digital Library     |     IEEE Standards     |     IEEE Spectrum     |     More Sites

Unverified Commit 85d1c869 authored by Edward W's avatar Edward W Committed by GitHub
Browse files

Import CSV (#69)

Import CSV Feature
parent b4e6a382
......@@ -626,6 +626,11 @@
"normalize-path": "^2.1.1"
}
},
"append-field": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY="
},
"argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
......@@ -843,6 +848,11 @@
"tweetnacl": "^0.14.3"
}
},
"bluebird": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"body-parser": {
"version": "1.19.0",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
......@@ -934,14 +944,45 @@
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"buffer-writer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
},
"busboy": {
"version": "0.2.14",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
"integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=",
"requires": {
"dicer": "0.2.5",
"readable-stream": "1.1.x"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
......@@ -1150,6 +1191,17 @@
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"requires": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
}
},
"content-disposition": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
......@@ -1197,8 +1249,7 @@
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cross-spawn": {
"version": "6.0.5",
......@@ -1236,6 +1287,26 @@
"cssom": "0.3.x"
}
},
"csvtojson": {
"version": "2.0.10",
"resolved": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz",
"integrity": "sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==",
"requires": {
"bluebird": "^3.5.1",
"lodash": "^4.17.3",
"strip-bom": "^2.0.0"
},
"dependencies": {
"strip-bom": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz",
"integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=",
"requires": {
"is-utf8": "^0.2.0"
}
}
}
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
......@@ -1367,6 +1438,38 @@
"integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=",
"dev": true
},
"dicer": {
"version": "0.2.5",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
"integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=",
"requires": {
"readable-stream": "1.1.x",
"streamsearch": "0.1.2"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"diff-sequences": {
"version": "24.9.0",
"resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz",
......@@ -1995,17 +2098,6 @@
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
"dev": true
},
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"formidable": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz",
......@@ -3060,6 +3152,11 @@
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
"dev": true
},
"is-utf8": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz",
"integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI="
},
"is-windows": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
......@@ -3075,8 +3172,7 @@
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isexe": {
"version": "2.0.0",
......@@ -3938,8 +4034,7 @@
"minimist": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
"dev": true
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
},
"mixin-deep": {
"version": "1.3.2",
......@@ -3966,7 +4061,6 @@
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"dev": true,
"requires": {
"minimist": "0.0.8"
}
......@@ -3976,6 +4070,21 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"multer": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.2.tgz",
"integrity": "sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==",
"requires": {
"append-field": "^1.0.0",
"busboy": "^0.2.11",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.1",
"object-assign": "^4.1.1",
"on-finished": "^2.3.0",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
}
},
"mute-stream": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
......@@ -4098,6 +4207,11 @@
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
"dev": true
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-copy": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
......@@ -4513,8 +4627,7 @@
"process-nextick-args": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
"dev": true
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
},
"progress": {
"version": "2.0.3",
......@@ -4615,7 +4728,6 @@
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"dev": true,
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
......@@ -4697,6 +4809,17 @@
"uuid": "^3.3.2"
},
"dependencies": {
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
......@@ -5284,6 +5407,11 @@
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=",
"dev": true
},
"streamsearch": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
},
"string-length": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/string-length/-/string-length-2.0.0.tgz",
......@@ -5328,7 +5456,6 @@
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"requires": {
"safe-buffer": "~5.1.0"
}
......@@ -5387,6 +5514,17 @@
"ms": "^2.1.1"
}
},
"form-data": {
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz",
"integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
......@@ -5615,6 +5753,11 @@
"mime-types": "~2.1.24"
}
},
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"uglify-js": {
"version": "3.6.9",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.9.tgz",
......@@ -5707,8 +5850,7 @@
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"util.promisify": {
"version": "1.0.0",
......
......@@ -14,10 +14,12 @@
"license": "ISC",
"dependencies": {
"body-parser": "^1.19.0",
"csvtojson": "^2.0.10",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"pg": "^7.12.1",
"lodash": ">=4.17.13"
"lodash": ">=4.17.13",
"multer": "^1.4.2",
"pg": "^7.12.1"
},
"devDependencies": {
"eslint": "^5.16.0",
......
......@@ -2,6 +2,10 @@ const request = require('supertest');
const app = require('../../app');
const { setup, teardown, loadSQL } = require('../setup');
const EXPECTED_ASSET_TYPE1 = { id: 1, name: 'Tree', description: null };
const EXPECTED_ASSET_TYPE2 = { id: 2, name: 'Lemur', description: 'Mammals of the order Primates, divided into 8 families and consisting of 15 genera and around 100 existing species. They are native only to the island of Madagascar.' };
const EXPECTED_ASSET_TYPE3 = { id: 3, name: 'Bison', description: 'Bison are large, even-toed ungulates in the genus Bison within the subfamily Bovinae.' };
describe('GET/POST assetDefinitions', () => {
beforeAll(async () => {
jest.setTimeout(30000);
......@@ -61,3 +65,51 @@ describe('GET/POST assetDefinitions', () => {
});
});
});
describe('GET assetTypes', () => {
beforeAll(async () => {
jest.setTimeout(30000);
await setup();
// create some default asset types
await loadSQL('../schema/sample-data-assetTypes.sql');
});
// Clean up after the tests are finished.
afterAll(async () => {
await teardown();
});
it('gets all asset types', () => {
return request(app)
.get('/api/v1/assetTypes')
.expect(200)
.then((res) => {
expect(res.body).toEqual(
expect.arrayContaining([
expect.objectContaining(EXPECTED_ASSET_TYPE1),
expect.objectContaining(EXPECTED_ASSET_TYPE2),
expect.objectContaining(EXPECTED_ASSET_TYPE3)
])
);
expect(res.body).toHaveLength(3);
});
});
});
// TODO - tests for PUT CSV endpoint
describe('PUT CSV', () => {
beforeAll(async () => {
jest.setTimeout(30000);
await setup();
// create some default asset definitions
await loadSQL('../schema/sample-data-assetTypes.sql');
});
// Clean up after the tests are finished.
afterAll(async () => {
await teardown();
});
});
const { findAssetTypes } = require('../assetDefinitions.controller');
const assetDefinitionsDb = require('../../db/assetDefinitions.db');
describe('assetDefinitions.controller.getAssetTypes', () => {
let req;
let res;
let next;
let expectedAssetTypes;
let data;
beforeEach(() => {
req = {};
res = {
json: jest.fn()
};
next = jest.fn();
expectedAssetTypes = [{}];
data = { rows: expectedAssetTypes };
assetDefinitionsDb.findAssetTypes = jest.fn(async () => data);
});
it('returns all asset types in DB', async () => {
await findAssetTypes(req, res, next);
expect(res.json).toHaveBeenCalledWith(expectedAssetTypes);
});
});
describe('assetDefinitions.controller.storeCSV', () => {
let req;
let res;
let next;
let assetTypeId;
let expected;
beforeEach(() => {
req = {
body: {},
file: {}
};
// Clear the response
res = {
json: jest.fn()
};
next = jest.fn();
expected = { success: true };
assetTypeId = 1;
assetDefinitionsDb.storeCSV = jest.fn(async () => expected);
});
// TODO - tests for storeCSV function in controller
});
const db = require('../db');
/**
* Maps asset definition related requests to the correct DB function and returns a response
*/
const find = async (req, res, next) => {
const db = require('../db');
const csv = require('csvtojson');
/**
* Gets all the different asset types from the database.
* @param {*} req - the request
* @param {*} res - the response
* @param {*} next - the next middleware function
*/
const getAssetTypes = async (req, res, next) => {
const predicates = req.query;
try {
const assetDefinitions = await db.assetDefinitions.find(predicates);
res.json(assetDefinitions);
const assetTypes = await db.assetDefinitions.findAssetTypes(predicates);
res.json(assetTypes);
} catch (e) {
next(e);
}
};
const create = async (req, res, next) => {
const assetDefinition = req.valid.assetDefinition;
/**
* Gets the property types for a given asset type.
* @param {*} req - the request
* @param {*} res - the response
* @param {*} next - the next middleware function
*/
const getAssetPropTypes = async (req, res, next) => {
try {
const assetTypeId = await db.assetDefinitions.create(assetDefinition);
res.json(assetTypeId);
const prop_types = await db.assetDefinitions.findAssetPropTypes(req.valid.assetTypeID);
res.json(prop_types);
} catch (e) {
next(e);
}
};
/**
* Gets all the different asset types from the database.
* @param {*} req - the request
* @param {*} res - the response
* @param {*} next - the next middleware function
*/
const getAssetTypes = async (req, res, next) => {
* Gets all the properties for all assets of a given asset type.
* @param {*} req - the request
* @param {*} res - the response
* @param {*} next - the next middleware function
*/
const getAssetPropsByTypeID = async (req, res, next) => {
try {
const data = await db.assetDefinitions.findAssetPropsByTypeID(req.valid.assetTypeID);
res.json(data);
} catch (e) {
next(e);
}
};
const find = async (req, res, next) => {
const predicates = req.query;
try {
const assetTypes = await db.assetDefinitions.findAssetTypes(predicates);
res.json(assetTypes);
const assetDefinitions = await db.assetDefinitions.find(predicates);
res.json(assetDefinitions);
} catch (e) {
next(e);
}
};
/**
* Gets the property types for a given asset type.
* @param {*} req - the request
* @param {*} res - the response
* @param {*} next - the next middleware function
*/
const getAssetPropTypes = async (req, res, next) => {
const create = async (req, res, next) => {
const assetDefinition = req.valid.assetDefinition;
try {
const prop_types = await db.assetDefinitions.findAssetPropTypes(req.valid.assetTypeID);
res.json(prop_types);
const assetTypeId = await db.assetDefinitions.create(assetDefinition);
res.json(assetTypeId);
} catch (e) {
next(e);
}
};
/**
* Gets all the properties for all assets of a given asset type.
* @param {*} req - the request
* @param {*} res - the response
* @param {*} next - the next middleware function
* Converts selected CSV file to JSON and adds the data to the DB
*
* @param {*} req The incoming Express request
* @param {*} res The outgoing Express request
* @param {*} next The next Express middleware function in the stack
*/
const getAssetPropsByTypeID = async (req, res, next) => {
const storeCSV = async(req, res, next) => {
const assetTypeId = req.body.assetTypeId;
const csvFile = req.file;
const csvPath = csvFile.path;
try {
const data = await db.assetDefinitions.findAssetPropsByTypeID(req.valid.assetTypeID);