Add support for binary vector (Oracle Database 23ai feature) and eslint fixes in code
This commit is contained in:
parent
0d72c1b271
commit
e4a261d78a
|
@ -93,12 +93,19 @@ async function run() {
|
|||
VCOL8 VECTOR(4, int8),
|
||||
VCOL VECTOR(4),
|
||||
VCOLFlexDouble VECTOR(*, *),
|
||||
VCOLFlexFloat VECTOR(*, *))`);
|
||||
VCOLFlexFloat VECTOR(*, *),
|
||||
VCOLBinary VECTOR(16, binary))`);
|
||||
|
||||
const arr = [2345.67, 12.2, -23.4, -65.2];
|
||||
const float64arr = new Float64Array(arr);
|
||||
const float32arr = new Float32Array(arr);
|
||||
const int8arr = new Int8Array([126, 125, -126, -23]);
|
||||
|
||||
// 16 dimensions numbering from left/msb
|
||||
// 1,1,1,1,0,0,0,0
|
||||
// 1,1,0,0,1,0,0,0
|
||||
const uInt8Arr = new Uint8Array([240, 200]);
|
||||
|
||||
// Add both float32 and float64 range elements
|
||||
const arrFlexDouble = [2345.67, 12.666428727762776];
|
||||
// Add only float32 range elements
|
||||
|
@ -106,20 +113,22 @@ async function run() {
|
|||
|
||||
console.log('Inserting Vector ');
|
||||
result = await connection.execute(`insert into ${tableName} values(:id, :vec32, :vec64, :vec8, :vec,
|
||||
:vecFlexDouble, :vecFlexFloat)`,
|
||||
:vecFlexDouble, :vecFlexFloat, :vecBinary)`,
|
||||
{ id: 1,
|
||||
vec32: float32arr,
|
||||
vec64: float64arr,
|
||||
vec8: int8arr,
|
||||
vec: {type: oracledb.DB_TYPE_VECTOR, val: arr},
|
||||
vecFlexDouble: {type: oracledb.DB_TYPE_VECTOR, val: arrFlexDouble},
|
||||
vecFlexFloat: {type: oracledb.DB_TYPE_VECTOR, val: arrFlexFloat32}
|
||||
vecFlexFloat: {type: oracledb.DB_TYPE_VECTOR, val: arrFlexFloat32},
|
||||
vecBinary: {type: oracledb.DB_TYPE_VECTOR, val: uInt8Arr}
|
||||
});
|
||||
console.log('Rows inserted: ' + result.rowsAffected);
|
||||
|
||||
console.log('Query Results:');
|
||||
result = await connection.execute(
|
||||
`select id, VCOL32, VCOL64, VCOL8, VCOL, VCOLFlexDouble, VCOLFlexFloat from ${tableName} ORDER BY id`);
|
||||
`select id, VCOL32, VCOL64, VCOL8, VCOL, VCOLFlexDouble, VCOLFlexFloat, VCOLBinary
|
||||
from ${tableName} ORDER BY id`);
|
||||
console.log("Query metadata:", result.metaData);
|
||||
console.log("Query rows:", result.rows);
|
||||
const vec32 = result.rows[0].VCOL32;
|
||||
|
@ -128,6 +137,7 @@ async function run() {
|
|||
const vec = result.rows[0].VCOL;
|
||||
const vecFlexDouble = result.rows[0].VCOLFLEXDOUBLE;
|
||||
const vecFlexFloat = result.rows[0].VCOLFLEXFLOAT;
|
||||
const vecBinary = result.rows[0].VCOLBINARY;
|
||||
|
||||
assert(vec32.constructor, Array);
|
||||
assert(vec64.constructor, Array);
|
||||
|
@ -135,6 +145,7 @@ async function run() {
|
|||
assert(vecFlexDouble.constructor, Array);
|
||||
assert(vecFlexFloat.constructor, Array);
|
||||
assert(vec8.constructor, Array);
|
||||
assert(vecBinary.constructor, Array);
|
||||
|
||||
// Reading vector as string.
|
||||
console.log("Fetch Vector Column as string");
|
||||
|
|
|
@ -165,4 +165,3 @@ class AzureProvider extends base {
|
|||
}
|
||||
}
|
||||
module.exports = AzureProvider;
|
||||
|
||||
|
|
|
@ -97,5 +97,3 @@ class base {
|
|||
|
||||
}
|
||||
module.exports = {base};
|
||||
|
||||
|
||||
|
|
|
@ -201,4 +201,3 @@ class OCIProvider extends base {
|
|||
}
|
||||
}
|
||||
module.exports = OCIProvider;
|
||||
|
||||
|
|
|
@ -202,9 +202,7 @@ class Connection extends EventEmitter {
|
|||
typeof value === 'boolean' ||
|
||||
typeof value === 'bigint' ||
|
||||
Array.isArray(value) ||
|
||||
value instanceof Float32Array ||
|
||||
value instanceof Float64Array ||
|
||||
value instanceof Int8Array ||
|
||||
nodbUtil.isVectorValue(value) ||
|
||||
Buffer.isBuffer(value) ||
|
||||
util.types.isDate(value) ||
|
||||
value instanceof Lob ||
|
||||
|
|
|
@ -205,5 +205,6 @@ module.exports = {
|
|||
VECTOR_FORMAT_FLOAT32: 2,
|
||||
VECTOR_FORMAT_FLOAT64: 3,
|
||||
VECTOR_FORMAT_INT8: 4,
|
||||
VECTOR_FORMAT_BINARY: 5,
|
||||
|
||||
};
|
||||
|
|
|
@ -32,7 +32,8 @@ module.exports = {
|
|||
|
||||
// vector constants
|
||||
TNS_VECTOR_MAGIC_BYTE: 0xDB,
|
||||
TNS_VECTOR_VERSION: 0,
|
||||
TNS_VECTOR_VERSION_BASE: 0,
|
||||
TNS_VECTOR_VERSION_WITH_BINARY: 1,
|
||||
|
||||
// vector flags
|
||||
TNS_VECTOR_FLAG_NORMSRC: 0x0010,
|
||||
|
@ -99,6 +100,7 @@ module.exports = {
|
|||
VECTOR_FORMAT_FLOAT32: constants.VECTOR_FORMAT_FLOAT32,
|
||||
VECTOR_FORMAT_FLOAT64: constants.VECTOR_FORMAT_FLOAT64,
|
||||
VECTOR_FORMAT_INT8: constants.VECTOR_FORMAT_INT8,
|
||||
VECTOR_FORMAT_BINARY: constants.VECTOR_FORMAT_BINARY,
|
||||
|
||||
TNS_NULL_LENGTH_INDICATOR: 255,
|
||||
TNS_LONG_LENGTH_INDICATOR: 254,
|
||||
|
|
|
@ -50,11 +50,11 @@ class VectorDecoder extends BaseBuffer {
|
|||
errors.throwErr(errors.ERR_UNEXPECTED_DATA,
|
||||
Buffer.from([magicByte]).toString('hex'));
|
||||
const version = this.readUInt8();
|
||||
if (version != constants.TNS_VECTOR_VERSION)
|
||||
if (version > constants.TNS_VECTOR_VERSION_WITH_BINARY)
|
||||
errors.throwErr(errors.ERR_VECTOR_VERSION_NOT_SUPPORTED, version);
|
||||
const flags = this.readUInt16BE();
|
||||
const vectorFormat = this.readUInt8();
|
||||
const numElements = this.readUInt32BE();
|
||||
let numElements = this.readUInt32BE();
|
||||
let elementSize, result;
|
||||
if (vectorFormat === constants.VECTOR_FORMAT_FLOAT32) {
|
||||
elementSize = 4;
|
||||
|
@ -65,6 +65,11 @@ class VectorDecoder extends BaseBuffer {
|
|||
} else if (vectorFormat === constants.VECTOR_FORMAT_INT8) {
|
||||
elementSize = 1;
|
||||
result = new Int8Array(numElements);
|
||||
} else if (vectorFormat === constants.VECTOR_FORMAT_BINARY) {
|
||||
elementSize = 1;
|
||||
// The number of dimensions are assumed to be multiple of 8.
|
||||
numElements = numElements / 8;
|
||||
result = new Uint8Array(numElements);
|
||||
} else {
|
||||
errors.throwErr(errors.ERR_VECTOR_FORMAT_NOT_SUPPORTED, vectorFormat);
|
||||
}
|
||||
|
@ -100,6 +105,8 @@ class VectorEncoder extends GrowableBuffer {
|
|||
// determine some basic information about the vector
|
||||
let vectorFormat = constants.VECTOR_FORMAT_FLOAT32;
|
||||
let writeFn = this.writeBinaryFloat.bind(this);
|
||||
let numElements = value.length;
|
||||
let vectorVersion = constants.TNS_VECTOR_VERSION_BASE;
|
||||
|
||||
if (Array.isArray(value) || value instanceof Float64Array) {
|
||||
vectorFormat = constants.VECTOR_FORMAT_FLOAT64;
|
||||
|
@ -107,6 +114,12 @@ class VectorEncoder extends GrowableBuffer {
|
|||
} else if (value instanceof Int8Array) {
|
||||
vectorFormat = constants.VECTOR_FORMAT_INT8;
|
||||
writeFn = this.writeSB1.bind(this);
|
||||
} else if (value.constructor.name === 'Uint8Array') {
|
||||
vectorFormat = constants.VECTOR_FORMAT_BINARY;
|
||||
// The number of dimensions are assumed to be multiple of 8.
|
||||
numElements = numElements * 8;
|
||||
vectorVersion = constants.TNS_VECTOR_VERSION_WITH_BINARY;
|
||||
writeFn = this.writeUInt8.bind(this);
|
||||
}
|
||||
|
||||
// Let server generate the norm (TNS_VECTOR_FLAG_NORMSRC)
|
||||
|
@ -115,10 +128,10 @@ class VectorEncoder extends GrowableBuffer {
|
|||
|
||||
// write header
|
||||
this.writeUInt8(constants.TNS_VECTOR_MAGIC_BYTE);
|
||||
this.writeUInt8(constants.TNS_VECTOR_VERSION);
|
||||
this.writeUInt8(vectorVersion);
|
||||
this.writeUInt16BE(flags);
|
||||
this.writeUInt8(vectorFormat);
|
||||
this.writeUInt32BE(value.length);
|
||||
this.writeUInt32BE(numElements);
|
||||
this.reserveBytes(8);
|
||||
|
||||
// write data
|
||||
|
|
|
@ -118,6 +118,8 @@ class Capabilities {
|
|||
constants.TNS_CCAP_CTB_IMPLICIT_POOL;
|
||||
this.compileCaps[constants.TNS_CCAP_TTC5] =
|
||||
constants.TNS_CCAP_VECTOR_SUPPORT;
|
||||
this.compileCaps[constants.TNS_CCAP_VECTOR_FEATURES] =
|
||||
constants.TNS_CCAP_VECTOR_FEATURE_BINARY;
|
||||
}
|
||||
|
||||
initRuntimeCaps() {
|
||||
|
|
|
@ -580,7 +580,8 @@ module.exports = {
|
|||
TNS_CCAP_TTC4: 40,
|
||||
TNS_CCAP_LOB2: 42,
|
||||
TNS_CCAP_TTC5: 44,
|
||||
TNS_CCAP_MAX: 51,
|
||||
TNS_CCAP_VECTOR_FEATURES: 52,
|
||||
TNS_CCAP_MAX: 53,
|
||||
|
||||
// compile time capability values
|
||||
TNS_CCAP_SQL_VERSION_MAX: 6,
|
||||
|
@ -636,6 +637,7 @@ module.exports = {
|
|||
TNS_CCAP_LOB2_2GB_PREFETCH: 0x04,
|
||||
TNS_CCAP_CTB_IMPLICIT_POOL: 0x08,
|
||||
TNS_CCAP_VECTOR_SUPPORT: 0x08,
|
||||
TNS_CCAP_VECTOR_FEATURE_BINARY: 0x01,
|
||||
|
||||
// runtime capability indices
|
||||
TNS_RCAP_COMPAT: 0,
|
||||
|
|
|
@ -228,8 +228,7 @@ function transformValueIn(info, value, options) {
|
|||
return value._impl;
|
||||
|
||||
// handle vectors
|
||||
} else if (value instanceof Float32Array || value instanceof Float64Array ||
|
||||
value instanceof Int8Array) {
|
||||
} else if (nodbUtil.isVectorValue(value)) {
|
||||
checkType(info, options, types.DB_TYPE_VECTOR);
|
||||
return value;
|
||||
} else if (info.type === types.DB_TYPE_VECTOR && Array.isArray(value)) {
|
||||
|
|
|
@ -393,7 +393,8 @@ function addTypeProperties(obj, attrName) {
|
|||
function isVectorValue(value) {
|
||||
return (value instanceof Float32Array ||
|
||||
value instanceof Float64Array ||
|
||||
value instanceof Int8Array);
|
||||
value instanceof Int8Array || (Object.getPrototypeOf(value)
|
||||
=== Uint8Array.prototype));
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
|
|
@ -859,6 +859,12 @@ bool njsBaton_getVectorValue(njsBaton *baton, dpiVector *vector,
|
|||
type = napi_int8_array;
|
||||
elementLength = 1;
|
||||
break;
|
||||
case DPI_VECTOR_FORMAT_BINARY:
|
||||
type = napi_uint8_array;
|
||||
elementLength = 1;
|
||||
// dimensions for binary is assumed to be multiples of 8.
|
||||
numElem = numElem / 8;
|
||||
break;
|
||||
default:
|
||||
return njsBaton_setErrorUnsupportedVectorFormat
|
||||
(baton, vectorInfo.format);
|
||||
|
|
|
@ -141,6 +141,7 @@ static bool njsJsonBuffer_populateNode(njsJsonBuffer *buf, dpiJsonNode *node,
|
|||
napi_env env, napi_value value, njsBaton *baton)
|
||||
{
|
||||
napi_value temp, name, fieldNames, fieldValues, vectorVal, global;
|
||||
napi_value uint8Val, uint8Proto, valProto;
|
||||
napi_valuetype valueType;
|
||||
napi_typedarray_type type;
|
||||
size_t tempBufferLength;
|
||||
|
@ -222,14 +223,24 @@ static bool njsJsonBuffer_populateNode(njsJsonBuffer *buf, dpiJsonNode *node,
|
|||
// handle vectors
|
||||
NJS_CHECK_NAPI(env, napi_is_typedarray(env, value, &isTyped))
|
||||
if (isTyped) {
|
||||
// check for type as buffer is a typedarray of type napi_uint8_array.
|
||||
NJS_CHECK_NAPI(env, napi_get_global(env, &global))
|
||||
NJS_CHECK_NAPI(env, napi_get_named_property(env, global, "Uint8Array",
|
||||
&uint8Val))
|
||||
NJS_CHECK_NAPI(env, napi_get_named_property(env, uint8Val, "prototype",
|
||||
&uint8Proto))
|
||||
NJS_CHECK_NAPI(env, napi_get_prototype(env, value, &valProto))
|
||||
|
||||
// See if the value prototype matches with Uint8Array.
|
||||
// The Buffer and JsonId return true for Uint8Array instance, so
|
||||
// using prototype check to see if the value is instance of Uint8Array.
|
||||
NJS_CHECK_NAPI(env, napi_strict_equals(env, uint8Proto, valProto,
|
||||
&check))
|
||||
NJS_CHECK_NAPI(env, napi_get_typedarray_info(env, value, &type,
|
||||
NULL, NULL, NULL, NULL))
|
||||
if ((type == napi_float64_array) || (type == napi_float32_array)
|
||||
|| (type == napi_int8_array)) {
|
||||
|| (type == napi_int8_array) || (check)) {
|
||||
node->oracleTypeNum = DPI_ORACLE_TYPE_VECTOR;
|
||||
node->nativeTypeNum = DPI_NATIVE_TYPE_BYTES;
|
||||
NJS_CHECK_NAPI(env, napi_get_global(env, &global))
|
||||
NJS_CHECK_NAPI(env, napi_call_function(env, global,
|
||||
baton->jsEncodeVectorFn, 1, &value, &vectorVal))
|
||||
NJS_CHECK_NAPI(env, napi_get_buffer_info(env, vectorVal,
|
||||
|
|
|
@ -812,6 +812,10 @@ bool njsVariable_setScalarValue(njsVariable *var, uint32_t pos, napi_env env,
|
|||
case napi_int8_array:
|
||||
vectorInfo.format = DPI_VECTOR_FORMAT_INT8;
|
||||
break;
|
||||
case napi_uint8_array:
|
||||
vectorInfo.numDimensions = (uint32_t)numElem * 8;
|
||||
vectorInfo.format = DPI_VECTOR_FORMAT_BINARY;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -198,6 +198,13 @@ describe('244.dataTypeJson.js', function() {
|
|||
KeyInt8: new Int8Array([-123, 12, 123]),
|
||||
keyBuf: Buffer.from("A Raw")
|
||||
};
|
||||
const jsonVal21 = {
|
||||
KeyF32: new Float32Array([1, 2]),
|
||||
KeyF64: new Float64Array([-992.1, 994.3]),
|
||||
KeyInt8: new Int8Array([-123, 12, 123]),
|
||||
KeyBinary: new Uint8Array([240, 120]),
|
||||
keyBuf: Buffer.from("A Raw")
|
||||
};
|
||||
const binds = [
|
||||
[1, jsonVal1],
|
||||
[2, jsonVal2],
|
||||
|
@ -225,6 +232,9 @@ describe('244.dataTypeJson.js', function() {
|
|||
binds.push([19, jsonVal19]);
|
||||
binds.push([20, jsonVal20]);
|
||||
}
|
||||
if (testsUtil.isVectorBinaryRunnable) {
|
||||
binds.push([21, jsonVal21]);
|
||||
}
|
||||
binds.forEach((element, index) => {
|
||||
binds[index].push(connection.encodeOSON(element[1]));
|
||||
});
|
||||
|
|
|
@ -240,6 +240,21 @@ testsUtil.isJsonMetaDataRunnable = async function() {
|
|||
return true;
|
||||
};
|
||||
|
||||
testsUtil.isVectorBinaryRunnable = async function() {
|
||||
const clientVersion = testsUtil.getClientVersion();
|
||||
let serverVersion;
|
||||
try {
|
||||
const conn = await oracledb.getConnection(dbConfig);
|
||||
serverVersion = conn.oracleServerVersion;
|
||||
await conn.close();
|
||||
} catch (error) {
|
||||
console.log('Error in checking VECTOR binary prerequisites:\n', error);
|
||||
}
|
||||
|
||||
return (serverVersion >= 2305000000
|
||||
&& (oracledb.thin || clientVersion >= 2305000000));
|
||||
};
|
||||
|
||||
testsUtil.generateRandomPassword = function(length = 6) {
|
||||
let result = "";
|
||||
const choices = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
||||
|
|
Loading…
Reference in New Issue