Fix tokenizer jieba tag flow and debug logging
This commit is contained in:
372
libs/jieba-wasm/jieba_rs_wasm_bg.js
Normal file
372
libs/jieba-wasm/jieba_rs_wasm_bg.js
Normal file
@@ -0,0 +1,372 @@
|
||||
let wasm;
|
||||
export function __wbg_set_wasm(val) {
|
||||
wasm = val;
|
||||
}
|
||||
|
||||
|
||||
const heap = new Array(128).fill(undefined);
|
||||
|
||||
heap.push(undefined, null, true, false);
|
||||
|
||||
function getObject(idx) { return heap[idx]; }
|
||||
|
||||
let heap_next = heap.length;
|
||||
|
||||
function dropObject(idx) {
|
||||
if (idx < 132) return;
|
||||
heap[idx] = heap_next;
|
||||
heap_next = idx;
|
||||
}
|
||||
|
||||
function takeObject(idx) {
|
||||
const ret = getObject(idx);
|
||||
dropObject(idx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder;
|
||||
|
||||
let cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
||||
|
||||
cachedTextDecoder.decode();
|
||||
|
||||
let cachedUint8ArrayMemory0 = null;
|
||||
|
||||
function getUint8ArrayMemory0() {
|
||||
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
||||
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
||||
}
|
||||
return cachedUint8ArrayMemory0;
|
||||
}
|
||||
|
||||
function getStringFromWasm0(ptr, len) {
|
||||
ptr = ptr >>> 0;
|
||||
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
||||
}
|
||||
|
||||
function addHeapObject(obj) {
|
||||
if (heap_next === heap.length) heap.push(heap.length + 1);
|
||||
const idx = heap_next;
|
||||
heap_next = heap[idx];
|
||||
|
||||
heap[idx] = obj;
|
||||
return idx;
|
||||
}
|
||||
|
||||
function debugString(val) {
|
||||
// primitive types
|
||||
const type = typeof val;
|
||||
if (type == 'number' || type == 'boolean' || val == null) {
|
||||
return `${val}`;
|
||||
}
|
||||
if (type == 'string') {
|
||||
return `"${val}"`;
|
||||
}
|
||||
if (type == 'symbol') {
|
||||
const description = val.description;
|
||||
if (description == null) {
|
||||
return 'Symbol';
|
||||
} else {
|
||||
return `Symbol(${description})`;
|
||||
}
|
||||
}
|
||||
if (type == 'function') {
|
||||
const name = val.name;
|
||||
if (typeof name == 'string' && name.length > 0) {
|
||||
return `Function(${name})`;
|
||||
} else {
|
||||
return 'Function';
|
||||
}
|
||||
}
|
||||
// objects
|
||||
if (Array.isArray(val)) {
|
||||
const length = val.length;
|
||||
let debug = '[';
|
||||
if (length > 0) {
|
||||
debug += debugString(val[0]);
|
||||
}
|
||||
for(let i = 1; i < length; i++) {
|
||||
debug += ', ' + debugString(val[i]);
|
||||
}
|
||||
debug += ']';
|
||||
return debug;
|
||||
}
|
||||
// Test for built-in
|
||||
const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val));
|
||||
let className;
|
||||
if (builtInMatches.length > 1) {
|
||||
className = builtInMatches[1];
|
||||
} else {
|
||||
// Failed to match the standard '[object ClassName]'
|
||||
return toString.call(val);
|
||||
}
|
||||
if (className == 'Object') {
|
||||
// we're a user defined class or Object
|
||||
// JSON.stringify avoids problems with cycles, and is generally much
|
||||
// easier than looping through ownProperties of `val`.
|
||||
try {
|
||||
return 'Object(' + JSON.stringify(val) + ')';
|
||||
} catch (_) {
|
||||
return 'Object';
|
||||
}
|
||||
}
|
||||
// errors
|
||||
if (val instanceof Error) {
|
||||
return `${val.name}: ${val.message}\n${val.stack}`;
|
||||
}
|
||||
// TODO we could test for more things here, like `Set`s and `Map`s.
|
||||
return className;
|
||||
}
|
||||
|
||||
let WASM_VECTOR_LEN = 0;
|
||||
|
||||
const lTextEncoder = typeof TextEncoder === 'undefined' ? (0, module.require)('util').TextEncoder : TextEncoder;
|
||||
|
||||
let cachedTextEncoder = new lTextEncoder('utf-8');
|
||||
|
||||
const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
|
||||
? function (arg, view) {
|
||||
return cachedTextEncoder.encodeInto(arg, view);
|
||||
}
|
||||
: function (arg, view) {
|
||||
const buf = cachedTextEncoder.encode(arg);
|
||||
view.set(buf);
|
||||
return {
|
||||
read: arg.length,
|
||||
written: buf.length
|
||||
};
|
||||
});
|
||||
|
||||
function passStringToWasm0(arg, malloc, realloc) {
|
||||
|
||||
if (realloc === undefined) {
|
||||
const buf = cachedTextEncoder.encode(arg);
|
||||
const ptr = malloc(buf.length, 1) >>> 0;
|
||||
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
|
||||
WASM_VECTOR_LEN = buf.length;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
let len = arg.length;
|
||||
let ptr = malloc(len, 1) >>> 0;
|
||||
|
||||
const mem = getUint8ArrayMemory0();
|
||||
|
||||
let offset = 0;
|
||||
|
||||
for (; offset < len; offset++) {
|
||||
const code = arg.charCodeAt(offset);
|
||||
if (code > 0x7F) break;
|
||||
mem[ptr + offset] = code;
|
||||
}
|
||||
|
||||
if (offset !== len) {
|
||||
if (offset !== 0) {
|
||||
arg = arg.slice(offset);
|
||||
}
|
||||
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
|
||||
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
|
||||
const ret = encodeString(arg, view);
|
||||
|
||||
offset += ret.written;
|
||||
ptr = realloc(ptr, len, offset, 1) >>> 0;
|
||||
}
|
||||
|
||||
WASM_VECTOR_LEN = offset;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
let cachedDataViewMemory0 = null;
|
||||
|
||||
function getDataViewMemory0() {
|
||||
if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
|
||||
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
|
||||
}
|
||||
return cachedDataViewMemory0;
|
||||
}
|
||||
|
||||
function isLikeNone(x) {
|
||||
return x === undefined || x === null;
|
||||
}
|
||||
|
||||
function getArrayJsValueFromWasm0(ptr, len) {
|
||||
ptr = ptr >>> 0;
|
||||
const mem = getDataViewMemory0();
|
||||
const result = [];
|
||||
for (let i = ptr; i < ptr + 4 * len; i += 4) {
|
||||
result.push(takeObject(mem.getUint32(i, true)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* @param {string} text
|
||||
* @param {boolean | undefined} [hmm]
|
||||
* @returns {any[]}
|
||||
*/
|
||||
export function cut(text, hmm) {
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
wasm.cut(retptr, ptr0, len0, isLikeNone(hmm) ? 0xFFFFFF : hmm ? 1 : 0);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
var v2 = getArrayJsValueFromWasm0(r0, r1).slice();
|
||||
wasm.__wbindgen_free(r0, r1 * 4, 4);
|
||||
return v2;
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} text
|
||||
* @returns {any[]}
|
||||
*/
|
||||
export function cut_all(text) {
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
wasm.cut_all(retptr, ptr0, len0);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
var v2 = getArrayJsValueFromWasm0(r0, r1).slice();
|
||||
wasm.__wbindgen_free(r0, r1 * 4, 4);
|
||||
return v2;
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} text
|
||||
* @param {boolean | undefined} [hmm]
|
||||
* @returns {any[]}
|
||||
*/
|
||||
export function cut_for_search(text, hmm) {
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
wasm.cut_for_search(retptr, ptr0, len0, isLikeNone(hmm) ? 0xFFFFFF : hmm ? 1 : 0);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
var v2 = getArrayJsValueFromWasm0(r0, r1).slice();
|
||||
wasm.__wbindgen_free(r0, r1 * 4, 4);
|
||||
return v2;
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} text
|
||||
* @param {string} mode
|
||||
* @param {boolean | undefined} [hmm]
|
||||
* @returns {any[]}
|
||||
*/
|
||||
export function tokenize(text, mode, hmm) {
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
const ptr1 = passStringToWasm0(mode, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len1 = WASM_VECTOR_LEN;
|
||||
wasm.tokenize(retptr, ptr0, len0, ptr1, len1, isLikeNone(hmm) ? 0xFFFFFF : hmm ? 1 : 0);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
|
||||
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
|
||||
if (r3) {
|
||||
throw takeObject(r2);
|
||||
}
|
||||
var v3 = getArrayJsValueFromWasm0(r0, r1).slice();
|
||||
wasm.__wbindgen_free(r0, r1 * 4, 4);
|
||||
return v3;
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} word
|
||||
* @param {number | undefined} [freq]
|
||||
* @param {string | undefined} [tag]
|
||||
* @returns {number}
|
||||
*/
|
||||
export function add_word(word, freq, tag) {
|
||||
const ptr0 = passStringToWasm0(word, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
var ptr1 = isLikeNone(tag) ? 0 : passStringToWasm0(tag, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
var len1 = WASM_VECTOR_LEN;
|
||||
const ret = wasm.add_word(ptr0, len0, !isLikeNone(freq), isLikeNone(freq) ? 0 : freq, ptr1, len1);
|
||||
return ret >>> 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} sentence
|
||||
* @param {boolean | undefined} [hmm]
|
||||
* @returns {any[]}
|
||||
*/
|
||||
export function tag(sentence, hmm) {
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
const ptr0 = passStringToWasm0(sentence, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
wasm.tag(retptr, ptr0, len0, isLikeNone(hmm) ? 0xFFFFFF : hmm ? 1 : 0);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
var v2 = getArrayJsValueFromWasm0(r0, r1).slice();
|
||||
wasm.__wbindgen_free(r0, r1 * 4, 4);
|
||||
return v2;
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
}
|
||||
}
|
||||
|
||||
export function __wbindgen_object_drop_ref(arg0) {
|
||||
takeObject(arg0);
|
||||
};
|
||||
|
||||
export function __wbindgen_string_new(arg0, arg1) {
|
||||
const ret = getStringFromWasm0(arg0, arg1);
|
||||
return addHeapObject(ret);
|
||||
};
|
||||
|
||||
export function __wbindgen_object_clone_ref(arg0) {
|
||||
const ret = getObject(arg0);
|
||||
return addHeapObject(ret);
|
||||
};
|
||||
|
||||
export function __wbg_new_1e7c00339420672b() {
|
||||
const ret = new Object();
|
||||
return addHeapObject(ret);
|
||||
};
|
||||
|
||||
export function __wbindgen_number_new(arg0) {
|
||||
const ret = arg0;
|
||||
return addHeapObject(ret);
|
||||
};
|
||||
|
||||
export function __wbg_set_1754fb90457a8cce(arg0, arg1, arg2) {
|
||||
getObject(arg0)[takeObject(arg1)] = takeObject(arg2);
|
||||
};
|
||||
|
||||
export function __wbg_new_b44ab9ef6060dd36(arg0, arg1) {
|
||||
const ret = new Error(getStringFromWasm0(arg0, arg1));
|
||||
return addHeapObject(ret);
|
||||
};
|
||||
|
||||
export function __wbindgen_debug_string(arg0, arg1) {
|
||||
const ret = debugString(getObject(arg1));
|
||||
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len1 = WASM_VECTOR_LEN;
|
||||
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
|
||||
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
|
||||
};
|
||||
|
||||
export function __wbindgen_throw(arg0, arg1) {
|
||||
throw new Error(getStringFromWasm0(arg0, arg1));
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user