dnrops
/
node-oracledb
mirror of https://github.com/oracle/node-oracledb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for binary vector (Oracle Database 23ai feature) and eslint fixes in code

This commit is contained in:
Sharad Chandran R 2024-07-15 13:05:18 +05:30
parent 0d72c1b271
commit e4a261d78a
17 changed files with 95 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
examples/vectortype1.js
View File

@ -93,12 +93,19 @@ async function run() {
VCOL8 VECTOR(4, int8),
VCOL VECTOR(4),
VCOLFlexDouble VECTOR(*, *),
VCOLFlexFloat VECTOR(*, *))`);
VCOLFlexFloat VECTOR(*, *),
VCOLBinary VECTOR(16, binary))`);
const arr = [2345.67, 12.2, -23.4, -65.2];
const float64arr = new Float64Array(arr);
const float32arr = new Float32Array(arr);
const int8arr = new Int8Array([126, 125, -126, -23]);
// 16 dimensions numbering from left/msb
// 1,1,1,1,0,0,0,0
// 1,1,0,0,1,0,0,0
const uInt8Arr = new Uint8Array([240, 200]);
// Add both float32 and float64 range elements
const arrFlexDouble = [2345.67, 12.666428727762776];
// Add only float32 range elements
@ -106,20 +113,22 @@ async function run() {
console.log('Inserting Vector ');
result = await connection.execute(`insert into ${tableName} values(:id, :vec32, :vec64, :vec8, :vec,
:vecFlexDouble, :vecFlexFloat)`,
:vecFlexDouble, :vecFlexFloat, :vecBinary)`,
{ id: 1,
vec32: float32arr,
vec64: float64arr,
vec8: int8arr,
vec: {type: oracledb.DB_TYPE_VECTOR, val: arr},
vecFlexDouble: {type: oracledb.DB_TYPE_VECTOR, val: arrFlexDouble},
vecFlexFloat: {type: oracledb.DB_TYPE_VECTOR, val: arrFlexFloat32}
vecFlexFloat: {type: oracledb.DB_TYPE_VECTOR, val: arrFlexFloat32},
vecBinary: {type: oracledb.DB_TYPE_VECTOR, val: uInt8Arr}
});
console.log('Rows inserted: ' + result.rowsAffected);
console.log('Query Results:');
result = await connection.execute(
`select id, VCOL32, VCOL64, VCOL8, VCOL, VCOLFlexDouble, VCOLFlexFloat from ${tableName} ORDER BY id`);
`select id, VCOL32, VCOL64, VCOL8, VCOL, VCOLFlexDouble, VCOLFlexFloat, VCOLBinary
from ${tableName} ORDER BY id`);
console.log("Query metadata:", result.metaData);
console.log("Query rows:", result.rows);
const vec32 = result.rows[0].VCOL32;
@ -128,6 +137,7 @@ async function run() {
const vec = result.rows[0].VCOL;
const vecFlexDouble = result.rows[0].VCOLFLEXDOUBLE;
const vecFlexFloat = result.rows[0].VCOLFLEXFLOAT;
const vecBinary = result.rows[0].VCOLBINARY;
assert(vec32.constructor, Array);
assert(vec64.constructor, Array);
@ -135,6 +145,7 @@ async function run() {
assert(vecFlexDouble.constructor, Array);
assert(vecFlexFloat.constructor, Array);
assert(vec8.constructor, Array);
assert(vecBinary.constructor, Array);
// Reading vector as string.
console.log("Fetch Vector Column as string");

1
lib/configProviders/azure.js
View File

@ -165,4 +165,3 @@ class AzureProvider extends base {
}
}
module.exports = AzureProvider;

2
lib/configProviders/base.js
View File

@ -97,5 +97,3 @@ class base {
}
module.exports = {base};

1
lib/configProviders/ociobject.js
View File

@ -201,4 +201,3 @@ class OCIProvider extends base {
}
}
module.exports = OCIProvider;

4
lib/connection.js
View File

@ -202,9 +202,7 @@ class Connection extends EventEmitter {
typeof value === 'boolean' ||
typeof value === 'bigint' ||
Array.isArray(value) ||
value instanceof Float32Array ||
value instanceof Float64Array ||
value instanceof Int8Array ||
nodbUtil.isVectorValue(value) ||
Buffer.isBuffer(value) ||
util.types.isDate(value) ||
value instanceof Lob ||

1
lib/constants.js
View File

@ -205,5 +205,6 @@ module.exports = {
VECTOR_FORMAT_FLOAT32: 2,
VECTOR_FORMAT_FLOAT64: 3,
VECTOR_FORMAT_INT8: 4,
VECTOR_FORMAT_BINARY: 5,
};

4
lib/impl/datahandlers/constants.js
View File

@ -32,7 +32,8 @@ module.exports = {
// vector constants
TNS_VECTOR_MAGIC_BYTE: 0xDB,
TNS_VECTOR_VERSION: 0,
TNS_VECTOR_VERSION_BASE: 0,
TNS_VECTOR_VERSION_WITH_BINARY: 1,
// vector flags
TNS_VECTOR_FLAG_NORMSRC: 0x0010,
@ -99,6 +100,7 @@ module.exports = {
VECTOR_FORMAT_FLOAT32: constants.VECTOR_FORMAT_FLOAT32,
VECTOR_FORMAT_FLOAT64: constants.VECTOR_FORMAT_FLOAT64,
VECTOR_FORMAT_INT8: constants.VECTOR_FORMAT_INT8,
VECTOR_FORMAT_BINARY: constants.VECTOR_FORMAT_BINARY,
TNS_NULL_LENGTH_INDICATOR: 255,
TNS_LONG_LENGTH_INDICATOR: 254,

21
lib/impl/datahandlers/vector.js
View File

@ -50,11 +50,11 @@ class VectorDecoder extends BaseBuffer {
errors.throwErr(errors.ERR_UNEXPECTED_DATA,
Buffer.from([magicByte]).toString('hex'));
const version = this.readUInt8();
if (version != constants.TNS_VECTOR_VERSION)
if (version > constants.TNS_VECTOR_VERSION_WITH_BINARY)
errors.throwErr(errors.ERR_VECTOR_VERSION_NOT_SUPPORTED, version);
const flags = this.readUInt16BE();
const vectorFormat = this.readUInt8();
const numElements = this.readUInt32BE();
let numElements = this.readUInt32BE();
let elementSize, result;
if (vectorFormat === constants.VECTOR_FORMAT_FLOAT32) {
elementSize = 4;
@ -65,6 +65,11 @@ class VectorDecoder extends BaseBuffer {
} else if (vectorFormat === constants.VECTOR_FORMAT_INT8) {
elementSize = 1;
result = new Int8Array(numElements);
} else if (vectorFormat === constants.VECTOR_FORMAT_BINARY) {
elementSize = 1;
// The number of dimensions are assumed to be multiple of 8.
numElements = numElements / 8;
result = new Uint8Array(numElements);
} else {
errors.throwErr(errors.ERR_VECTOR_FORMAT_NOT_SUPPORTED, vectorFormat);
}
@ -100,6 +105,8 @@ class VectorEncoder extends GrowableBuffer {
// determine some basic information about the vector
let vectorFormat = constants.VECTOR_FORMAT_FLOAT32;
let writeFn = this.writeBinaryFloat.bind(this);
let numElements = value.length;
let vectorVersion = constants.TNS_VECTOR_VERSION_BASE;
if (Array.isArray(value) || value instanceof Float64Array) {
vectorFormat = constants.VECTOR_FORMAT_FLOAT64;
@ -107,6 +114,12 @@ class VectorEncoder extends GrowableBuffer {
} else if (value instanceof Int8Array) {
vectorFormat = constants.VECTOR_FORMAT_INT8;
writeFn = this.writeSB1.bind(this);
} else if (value.constructor.name === 'Uint8Array') {
vectorFormat = constants.VECTOR_FORMAT_BINARY;
// The number of dimensions are assumed to be multiple of 8.
numElements = numElements * 8;
vectorVersion = constants.TNS_VECTOR_VERSION_WITH_BINARY;
writeFn = this.writeUInt8.bind(this);
}
// Let server generate the norm (TNS_VECTOR_FLAG_NORMSRC)
@ -115,10 +128,10 @@ class VectorEncoder extends GrowableBuffer {
// write header
this.writeUInt8(constants.TNS_VECTOR_MAGIC_BYTE);
this.writeUInt8(constants.TNS_VECTOR_VERSION);
this.writeUInt8(vectorVersion);
this.writeUInt16BE(flags);
this.writeUInt8(vectorFormat);
this.writeUInt32BE(value.length);
this.writeUInt32BE(numElements);
this.reserveBytes(8);
// write data

2
lib/thin/protocol/capabilities.js
View File

@ -118,6 +118,8 @@ class Capabilities {
constants.TNS_CCAP_CTB_IMPLICIT_POOL;
this.compileCaps[constants.TNS_CCAP_TTC5] =
constants.TNS_CCAP_VECTOR_SUPPORT;
this.compileCaps[constants.TNS_CCAP_VECTOR_FEATURES] =
constants.TNS_CCAP_VECTOR_FEATURE_BINARY;
}
initRuntimeCaps() {

4
lib/thin/protocol/constants.js
View File

@ -580,7 +580,8 @@ module.exports = {
TNS_CCAP_TTC4: 40,
TNS_CCAP_LOB2: 42,
TNS_CCAP_TTC5: 44,
TNS_CCAP_MAX: 51,
TNS_CCAP_VECTOR_FEATURES: 52,
TNS_CCAP_MAX: 53,
// compile time capability values
TNS_CCAP_SQL_VERSION_MAX: 6,
@ -636,6 +637,7 @@ module.exports = {
TNS_CCAP_LOB2_2GB_PREFETCH: 0x04,
TNS_CCAP_CTB_IMPLICIT_POOL: 0x08,
TNS_CCAP_VECTOR_SUPPORT: 0x08,
TNS_CCAP_VECTOR_FEATURE_BINARY: 0x01,
// runtime capability indices
TNS_RCAP_COMPAT: 0,

3
lib/transformer.js
View File

@ -228,8 +228,7 @@ function transformValueIn(info, value, options) {
return value._impl;
// handle vectors
} else if (value instanceof Float32Array || value instanceof Float64Array ||
value instanceof Int8Array) {
} else if (nodbUtil.isVectorValue(value)) {
checkType(info, options, types.DB_TYPE_VECTOR);
return value;
} else if (info.type === types.DB_TYPE_VECTOR && Array.isArray(value)) {

3
lib/util.js
View File

@ -393,7 +393,8 @@ function addTypeProperties(obj, attrName) {
function isVectorValue(value) {
return (value instanceof Float32Array ||
value instanceof Float64Array ||
value instanceof Int8Array);
value instanceof Int8Array || (Object.getPrototypeOf(value)
=== Uint8Array.prototype));
}
//-----------------------------------------------------------------------------

6
src/njsBaton.c
View File

@ -859,6 +859,12 @@ bool njsBaton_getVectorValue(njsBaton *baton, dpiVector *vector,
type = napi_int8_array;
elementLength = 1;
break;
case DPI_VECTOR_FORMAT_BINARY:
type = napi_uint8_array;
elementLength = 1;
// dimensions for binary is assumed to be multiples of 8.
numElem = numElem / 8;
break;
default:
return njsBaton_setErrorUnsupportedVectorFormat
(baton, vectorInfo.format);

17
src/njsJsonBuffer.c
View File

@ -141,6 +141,7 @@ static bool njsJsonBuffer_populateNode(njsJsonBuffer *buf, dpiJsonNode *node,
napi_env env, napi_value value, njsBaton *baton)
{
napi_value temp, name, fieldNames, fieldValues, vectorVal, global;
napi_value uint8Val, uint8Proto, valProto;
napi_valuetype valueType;
napi_typedarray_type type;
size_t tempBufferLength;
@ -222,14 +223,24 @@ static bool njsJsonBuffer_populateNode(njsJsonBuffer *buf, dpiJsonNode *node,
// handle vectors
NJS_CHECK_NAPI(env, napi_is_typedarray(env, value, &isTyped))
if (isTyped) {
// check for type as buffer is a typedarray of type napi_uint8_array.
NJS_CHECK_NAPI(env, napi_get_global(env, &global))
NJS_CHECK_NAPI(env, napi_get_named_property(env, global, "Uint8Array",
&uint8Val))
NJS_CHECK_NAPI(env, napi_get_named_property(env, uint8Val, "prototype",
&uint8Proto))
NJS_CHECK_NAPI(env, napi_get_prototype(env, value, &valProto))
// See if the value prototype matches with Uint8Array.
// The Buffer and JsonId return true for Uint8Array instance, so
// using prototype check to see if the value is instance of Uint8Array.
NJS_CHECK_NAPI(env, napi_strict_equals(env, uint8Proto, valProto,
&check))
NJS_CHECK_NAPI(env, napi_get_typedarray_info(env, value, &type,
NULL, NULL, NULL, NULL))
if ((type == napi_float64_array) || (type == napi_float32_array)
|| (type == napi_int8_array)) {
|| (type == napi_int8_array) || (check)) {
node->oracleTypeNum = DPI_ORACLE_TYPE_VECTOR;
node->nativeTypeNum = DPI_NATIVE_TYPE_BYTES;
NJS_CHECK_NAPI(env, napi_get_global(env, &global))
NJS_CHECK_NAPI(env, napi_call_function(env, global,
baton->jsEncodeVectorFn, 1, &value, &vectorVal))
NJS_CHECK_NAPI(env, napi_get_buffer_info(env, vectorVal,

4
src/njsVariable.c
View File

@ -812,6 +812,10 @@ bool njsVariable_setScalarValue(njsVariable *var, uint32_t pos, napi_env env,
case napi_int8_array:
vectorInfo.format = DPI_VECTOR_FORMAT_INT8;
break;
case napi_uint8_array:
vectorInfo.numDimensions = (uint32_t)numElem * 8;
vectorInfo.format = DPI_VECTOR_FORMAT_BINARY;
break;
default:
break;
}

10
test/dataTypeJson.js
View File

@ -198,6 +198,13 @@ describe('244.dataTypeJson.js', function() {
KeyInt8: new Int8Array([-123, 12, 123]),
keyBuf: Buffer.from("A Raw")
};
const jsonVal21 = {
KeyF32: new Float32Array([1, 2]),
KeyF64: new Float64Array([-992.1, 994.3]),
KeyInt8: new Int8Array([-123, 12, 123]),
KeyBinary: new Uint8Array([240, 120]),
keyBuf: Buffer.from("A Raw")
};
const binds = [
[1, jsonVal1],
[2, jsonVal2],
@ -225,6 +232,9 @@ describe('244.dataTypeJson.js', function() {
binds.push([19, jsonVal19]);
binds.push([20, jsonVal20]);
}
if (testsUtil.isVectorBinaryRunnable) {
binds.push([21, jsonVal21]);
}
binds.forEach((element, index) => {
binds[index].push(connection.encodeOSON(element[1]));
});

15
test/testsUtil.js
View File

@ -240,6 +240,21 @@ testsUtil.isJsonMetaDataRunnable = async function() {
return true;
};
testsUtil.isVectorBinaryRunnable = async function() {
const clientVersion = testsUtil.getClientVersion();
let serverVersion;
try {
const conn = await oracledb.getConnection(dbConfig);
serverVersion = conn.oracleServerVersion;
await conn.close();
} catch (error) {
console.log('Error in checking VECTOR binary prerequisites:\n', error);
}
return (serverVersion >= 2305000000
&& (oracledb.thin || clientVersion >= 2305000000));
};
testsUtil.generateRandomPassword = function(length = 6) {
let result = "";
const choices = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";