IEEE.org     |     IEEE Xplore Digital Library     |     IEEE Standards     |     IEEE Spectrum     |     More Sites

Unverified Commit 09448e6c authored by tilley14's avatar tilley14 Committed by GitHub
Browse files

Temportal queries (#76)

(Edward Wong and Nikolas Tilley)

Added Geo-Spacial and Temporal Query Endpoints to the Open Source DB
Added History table to the database
Asset Properties get saved in the History Table
parent 334e32fd
......@@ -3896,8 +3896,7 @@
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
"dev": true
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
},
"lodash.sortby": {
"version": "4.7.0",
......@@ -4065,6 +4064,11 @@
"minimist": "0.0.8"
}
},
"moment": {
"version": "2.24.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
......
......@@ -18,6 +18,7 @@
"dotenv": "^8.2.0",
"express": "^4.17.1",
"lodash": ">=4.17.13",
"moment": "^2.24.0",
"multer": "^1.4.2",
"pg": "^7.12.1"
},
......
......@@ -13,6 +13,7 @@ const POLYGON_ENDPOINT = `${GEOMETRY_ENDPOINT}/polygon`;
describe('GET assets/geometrySearch/envelope', () => {
beforeAll(async () => {
jest.setTimeout(30000);
await setup();
await loadSQL('../schema/sample-data-emptyProjects.sql');
});
......@@ -63,6 +64,7 @@ describe('GET assets/geometrySearch/envelope', () => {
describe('GET assets/geometrySearch/distance', () => {
beforeAll(async () => {
jest.setTimeout(30000);
await setup();
await loadSQL('../schema/sample-data-emptyProjects.sql');
});
......@@ -94,6 +96,7 @@ describe('GET assets/geometrySearch/polygon', () => {
let pack = (lat, lon) => { return { latitude: lat, longitude: lon}; };
beforeAll(async () => {
jest.setTimeout(30000);
await setup();
await loadSQL('../schema/sample-data-emptyProjects.sql');
});
......
const request = require('supertest');
const app = require('../../app');
const { setup, teardown, loadSQL } = require('../setup');
const ENDPOINT = '/api/v1/assets/properties/temporalSearch';
describe('GET assets/properties/temporalSearch', () => {
const polygon_search = {
'geometry': {
'type': 'Polygon',
'coordinates' : [[-16, 44], [-18, 44], [-18, 48], [-16, 48], [-16, 44]]
},
'sponsor': 'seneca park zoo society',
'asset_type': 'fire',
'project': 'Madagascar reforesting project'
};
const circle_search = {
'geometry': {
'type': 'Circle',
'coordinates' : [-16, 44],
'radius': '10000'
},
'start_date': '2020-05-20',
'end_date': '2020-05-25'
};
beforeAll(async () => {
jest.setTimeout(30000);
await setup();
await loadSQL('../schema/sample-data-emptyProjects.sql');
});
afterAll(async () => {
await teardown();
});
afterEach(async () => {
await global.dbPool.query('DELETE FROM asset');
});
it('returns HTTP 200 with "Polygon" search', async () => {
await request(app)
.get(ENDPOINT)
.send(polygon_search)
.expect(200);
});
it('returns HTTP 200 with "Circle" search', async () => {
await request(app)
.get(ENDPOINT)
.send(circle_search)
.expect(200);
});
});
const { temporalSearch } = require('../temporal.controller');
const temporalDb = require('../../db/temporal.db');
const moment = require('moment');
describe('temporal.controller.temporalSearch', () => {
let req;
let res;
let next;
let expected;
beforeEach(() => {
req = {
valid: {geometry: {coordinates: [1, 2], type: 'Circle', radius: 500}},
query: {}
};
res = {
json: jest.fn(),
send: jest.fn(),
status: jest.fn(() => res)
};
next = jest.fn();
expected = [{}];
temporalDb.temporalSearch = jest.fn(async () => expected);
});
it('jsonifies results', async () => {
await temporalSearch(req, res, next);
expect(temporalDb.temporalSearch).toHaveBeenCalledWith(
req.valid.geometry, undefined, undefined, undefined, undefined, undefined, undefined
);
expect(res.json).toHaveBeenCalledWith(expected);
});
it('accesses DB with valid parameters', async () => {
const validAssetId = 1;
const validSponsorName = 'SponsorName';
const validProjectName = 'ProjectName';
const validAssetTypeName = 'AssetTypeName';
const validStartDate = moment('2020-05-20', 'YYYY-MM-DD', true);
const validEndDate = moment('2020-05-22', 'YYYY-MM-DD', true);
req.valid['asset_id'] = validAssetId;
req.valid['sponsor_name'] = validSponsorName;
req.valid['project_name'] = validProjectName;
req.valid['asset_type_name'] = validAssetTypeName;
req.valid['start_date'] = validStartDate;
req.valid['end_date'] = validEndDate;
await temporalSearch(req, res, next);
expect(temporalDb.temporalSearch).toHaveBeenCalledWith(
req.valid.geometry,
validAssetId,
validSponsorName,
validProjectName,
validAssetTypeName,
validStartDate,
validEndDate
);
expect(res.json).toHaveBeenCalledWith(expected);
});
it('catches DB access exceptions and passes errors to the Express error handler', async () => {
const DB_ERROR = new Error();
temporalDb.temporalSearch = jest.fn(async () => { throw DB_ERROR; });
await temporalSearch(req, res, next);
expect(next).toHaveBeenCalledWith(DB_ERROR);
expect(res.json).not.toHaveBeenCalled();
});
});
......@@ -4,6 +4,7 @@ const bboxAssets = require('./bboxAssets.controller');
const dataTypes = require('./dataTypes.controller');
const geometrySearch = require('./geometrySearch.controller');
const projects = require('./projects.controller');
const temporal = require('./temporal.controller');
module.exports = {
assets,
......@@ -11,5 +12,6 @@ module.exports = {
bboxAssets,
dataTypes,
geometrySearch,
projects
projects,
temporal
};
/**
* Maps Temporal Request Params to the correct db querry
*/
const temporalDb = require('../db/temporal.db');
const temporalSearch = async (req, res, next) => {
const asset_id = req.valid.asset_id;
const sponsor_name = req.valid.sponsor_name;
const project_name = req.valid.project_name;
const asset_type_name = req.valid.asset_type_name;
const start_date = req.valid.start_date;
const end_date = req.valid.end_date;
const geometry = req.valid.geometry;
try {
const ret = await temporalDb.temporalSearch(geometry, asset_id, sponsor_name, project_name,
asset_type_name, start_date, end_date);
res.json(ret);
} catch (e) {
next(e);
}
};
module.exports = {
temporalSearch
};
/**
* Tests for Asset Definitions database layer
*/
const { findAssetTypes, findAssetPropTypes, findAsset,
findAssetProperty, createAssetProperty, updateAssetProperty } = require('../assetDefinitions.db');
const { findAssetTypes, findAssetPropTypes, findAsset, findAssetProperty,
createAssetProperty, addAssetPropertyToHistory, updateAssetProperty } = require('../assetDefinitions.db');
describe('assetDefinitions.db.findAssetTypes', () => {
let rows;
......@@ -102,6 +102,31 @@ describe('assetDefinitions.db.createAssetProperty', () => {
});
});
describe('assetDefinitions.db.addAssetPropertyToHistory', () => {
let rows;
let query;
let release;
let client;
beforeEach(() => {
rows = [{}];
query = jest.fn(async () => ({ rows }));
release = jest.fn(() => { return undefined; });
client = { query, release };
global.dbPool.connect = jest.fn(async () => { return client; });
});
it('adds asset properties to the history table', async() => {
const assetId = 1;
const propertyId = 1;
const value = 1;
const date = new Date();
await addAssetPropertyToHistory(client, assetId, propertyId, value, date);
expect(query).toHaveBeenCalledTimes(1);
});
});
describe('assetDefinitions.db.updateAssetProperty', () => {
let rows;
let query;
......
const { temporalSearch } = require('../temporal.db');
const moment = require('moment');
describe('temporal.db.temporalSearch', () => {
let query;
let rows;
let geometry;
beforeEach(() => {
rows = [];
query = jest.fn(async () => ({ rows }));
global.dbPool = { query };
geometry = {coordinates: [1, 2], type: 'Circle', radius: 500};
});
it('performs a ST_DWithin query when given a "Circle" geometry type', async () => {
await temporalSearch(geometry, undefined, undefined, undefined, undefined, undefined, undefined);
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('ST_DWithin'));
expect(query.mock.calls[0][1]).toEqual(expect.arrayContaining(
[1, 2, 500]
));
});
it('performs a ST_Within query when given a "Polygon" geometry type', async () => {
geometry = {coordinates: [[1, 1], [2, 2], [3, 3], [1, 1]], type: 'Polygon'};
await temporalSearch(geometry, undefined, undefined, undefined, undefined, undefined, undefined);
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('ST_Within'));
expect(query.mock.calls[0][1]).toEqual(expect.arrayContaining(
['LINESTRING(1 1,2 2,3 3,1 1)']
));
});
it('performs a query for asset properties after a start_date', async () => {
const start_string = '2012-05-25';
const start = moment(start_string, 'YYYY-MM-DD', true);
await temporalSearch(geometry, undefined, undefined, undefined, undefined, start, undefined);
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('history.date >='));
expect(query.mock.calls[0][1]).toEqual(expect.arrayContaining(
[start_string]
));
});
it('performs a query for asset properties before an end_date', async () => {
const end_string = '2012-05-25'
const end = moment(end_string, 'YYYY-MM-DD', true);
await temporalSearch(geometry, undefined, undefined, undefined, undefined, undefined, end);
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('history.date <='));
expect(query.mock.calls[0][1]).toEqual(expect.arrayContaining(
[end_string]
));
});
it('performs a query for asset properties between a start_date and an end_date', async () => {
const start_string = '2012-05-20';
const start = moment(start_string, 'YYYY-MM-DD', true);
const end_string = '2012-05-25'
const end = moment(end_string, 'YYYY-MM-DD', true);
await temporalSearch(geometry, undefined, undefined, undefined, undefined, start, end);
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('history.date BETWEEN'));
expect(query.mock.calls[0][1]).toEqual(expect.arrayContaining(
[start_string, end_string]
));
});
it('will further filter queries for sponsors, asset types, and project names', async () => {
const sponsor_name = 'SponsorName';
const asset_type = 'AssetType';
const project_name = 'ProjectName';
await temporalSearch(geometry, undefined, sponsor_name, asset_type, project_name, undefined, undefined);
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('LOWER(sponsor.name)'));
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('LOWER(project.name)'));
expect(query.mock.calls[0][0]).toEqual(expect.stringContaining('LOWER(asset_type.name)'));
expect(query.mock.calls[0][1]).toEqual(expect.arrayContaining(
[sponsor_name, asset_type, project_name]
));
});
it('formats query results as GeoJson', async () => {
rows = [{
asset_id: 1,
asset_type: 'AssetType',
property: 'PropertyName',
value: 'PropertyValue',
date: '2020-05-20',
sponsor_name: 'SponsorName',
project_name: 'ProjectName',
longitude: 1,
latitude: 2
}];
const expectedGeoJson = {
'type': 'FeatureCollection',
'features': [{
'type': 'Feature',
'geometry' : {
'type' : 'Point',
'coordinates' : [1, 2]
},
'properties': {
'asset_id' : 1,
'asset_type' : 'AssetType',
'asset_properties' : [{'property': 'PropertyName', 'value': 'PropertyValue'}],
'sponsor_name': 'SponsorName',
'project_name': 'ProjectName',
'date': '2020-05-20',
}
}]
};
const results = await temporalSearch(geometry, undefined, undefined, undefined, undefined, undefined, undefined);
await expect(results).toEqual(expect.objectContaining(expectedGeoJson));
});
});
......@@ -34,6 +34,7 @@ const findAssetTypes = async () => {
description
FROM
asset_type
ORDER BY id
`;
return global.dbPool.query(query);
......@@ -173,23 +174,6 @@ const create = async (assetDefinition) => {
}
};
// /**
// * Gets all properties associated with an asset type using the asset type's ID
// *
// * @param {Number} assetTypeId ID of the asset type whose properties are being queried
// */
// const findPropertiesByAssetTypeId = async(assetTypeId) => {
// let query = PROPERTIES_QUERY;
// const values = [];
// if ((typeof assetTypeId !== 'undefined') & (assetTypeId > 0)) {
// values.push(assetTypeId);
// query = query + ` WHERE asset_type_id = $${values.length}`;
// }
// return global.dbPool.query(query, values);
// };
/**
* Gets the asset associated given with the asset ID
*
......@@ -257,6 +241,22 @@ const createAssetProperty = async (client, assetId, propertyId, value) => {
return client.query(query, values);
};
const addAssetPropertyToHistory = async (client, assetId, propertyId, value, date) => {
// Generate the SQL command
const query = `
INSERT INTO history
(asset_id, property_id, value, date)
VALUES
($1, $2, $3, $4)
`;
// Generate the values to subsitute into the SQL command
const values = [assetId, propertyId, value, date];
// Execute the SQL command
return client.query(query, values);
};
/**
* Updates an asset property that is already stored in the DB
*
......@@ -361,16 +361,19 @@ const storeCSV = async(assetTypeId, csvJson) => {
value = asset[propertyName];
let assetProperties = (await findAssetProperty(assetId, propertyId)).rows;
let date = new Date();
// Throw an error if a row fails to contain a value for a property that is required
if (value === '' && propertyIsRequired) {
throw 'The selected CSV file is missing a required value (' + propertyName + ', ' + JSON.stringify(asset) + ')';
}
else if (assetProperties.length > 0) {
if (assetProperties[0].value !== value) {
await addAssetPropertyToHistory(client, assetId, propertyId, value, date);
await updateAssetProperty(client, assetId, propertyId, value);
}
}
else {
await addAssetPropertyToHistory(client, assetId, propertyId, value, date);
await createAssetProperty(client, assetId, propertyId, value);
}
}
......@@ -398,6 +401,7 @@ module.exports = {
findAsset,
findAssetProperty,
createAssetProperty,
addAssetPropertyToHistory,
updateAssetProperty,
storeCSV
};
......@@ -4,6 +4,7 @@ const bboxAssets = require('./bboxAssets.db');
const dataTypes = require('./dataTypes.db');
const geometrySearch = require('./geometrySearch.db');
const projects = require('./projects.db');
const temporal = require('./temporal.db');
module.exports = {
......@@ -12,5 +13,6 @@ module.exports = {
bboxAssets,
dataTypes,
geometrySearch,
projects
projects,
temporal
};
const utils = require('../utils');
const QUERY_HISTORY = `
SELECT
asset.id AS asset_id,
asset_type.name AS asset_type,
property.name AS property,
history.value AS value,
history.date AS date,
sponsor.name AS sponsor_name,
project.name AS project_name,
ST_X(asset.location) AS longitude,
ST_Y(asset.location) AS latitude
FROM
asset
JOIN history ON asset.id = history.asset_id
JOIN property ON history.property_id = property.id
JOIN asset_type ON asset.asset_type_id = asset_type.id
JOIN project ON asset.project_id = project.id
JOIN sponsor ON project.sponsor_id = sponsor.id
WHERE
TRUE
`;
const ORDER_BY = `
ORDER BY
asset.id ASC,
history.date DESC
`;
const D_WITHIN = `
ST_DWithin(ST_SetSRID(asset.location, 4326)::geography,
ST_SetSRID(ST_MakePoint($1, $2), 4326)::geography,
$3)
`;
const POLYGON_WITHIN = `
ST_Within(asset.location, ST_MakePolygon(ST_GeomFromText($1)))
`;
/**
* Searches the history table for asset properties that fall within specific search parameters
*
* @param {geometry} geometry - a GeoJson geometry
* @param {number} asset_id - A specific asset's id
* @param {string} sponsor_name - A sponsor's id
* @param {string} project_name - A projects's id
* @param {string} asset_type_name - An asset type's id
* @param {moment} start_date - A temporal lower bound of the history search
* @param {moment} end_date - A temporal upper bound of the history search
*
* Using GeoJSON as a template https://tools.ietf.org/html/rfc7946
* @returns {FeatureCollection} GeoJson - the historic properties of assets that meet the search parameters formatted following the GeoJSON standard.
*
*/
const temporalSearch = async (geometry, asset_id, sponsor_name, project_name, asset_type_name, start_date, end_date) => {
let query = QUERY_HISTORY;
let values = [];
switch (geometry.type) {
case 'Circle':
values.push(geometry.coordinates[0]);
values.push(geometry.coordinates[1]);
values.push(geometry.radius);
query += ' AND ' + D_WITHIN + ' ';
break;
case 'Polygon':
values.push(utils.db.makeLineStringFromGeoJsonCoordinates(geometry.coordinates));
query += ' AND ' + POLYGON_WITHIN + ' ';
break;
}
if ((typeof asset_id !== 'undefined') && (asset_id > 0)) {
values.push(asset_id);
query += ` AND asset.id = $${values.length}` + ' ';
}
if ((typeof sponsor_name !== 'undefined') && (sponsor_name !== '')) {
values.push(sponsor_name);
query += ` AND LOWER(sponsor.name) = LOWER($${values.length})` + ' ';
}
if ((typeof project_name !== 'undefined') && (project_name !== '')) {
values.push(project_name);
query += ` AND LOWER(project.name) = LOWER($${values.length})` + ' ';
}
if ((typeof asset_type_name !== 'undefined') && (asset_type_name !== '')) {
values.push(asset_type_name);
query += ` AND LOWER(asset_type.name) = LOWER($${values.length})` + ' ';
}
if ((typeof start_date !== 'undefined') || (typeof end_date !== 'undefined')) {
const has_start_date = (typeof start_date !== 'undefined');
const has_end_date = (typeof end_date !== 'undefined');
if (has_start_date && has_end_date) {
if (start_date.isValid() && end_date.isValid()) {
values.push(start_date.format(utils.shared.dateStringFormat()));
const idx_start = values.length;