diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 9b7acc42a..af09264f3 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -82,7 +82,7 @@ static bool try_append_ident(const char *source, uint32_t s, int len, uint32_t * /* Walk AST body, collect unique identifier text as space-separated string. * Returns arena-allocated string or NULL. */ static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) { - enum { BT_STACK = 512, BT_BUF = 512, BT_MAX_IDENTS = 40, BT_SEEN = 128, BT_SEEN_MASK = 127 }; + enum { BT_STACK = 512, BT_BUF = 2048, BT_MAX_IDENTS = 128, BT_SEEN = 256, BT_SEEN_MASK = 255 }; TSNode bt_stack[BT_STACK]; int bt_top = 0; bt_stack[bt_top++] = body; @@ -98,7 +98,11 @@ static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) { if (nc == 0) { const char *k = ts_node_type(nd); if (strcmp(k, "identifier") == 0 || strcmp(k, "field_identifier") == 0 || - strcmp(k, "property_identifier") == 0) { + strcmp(k, "property_identifier") == 0 || strcmp(k, "type_identifier") == 0 || + strcmp(k, "objectscript_identifier_special") == 0 || + strcmp(k, "identifier_segment_immediate_special") == 0 || + strcmp(k, "class_name") == 0 || strcmp(k, "method_name") == 0 || + strcmp(k, "routine_name") == 0 || strcmp(k, "quote_permitting_identifier") == 0) { uint32_t s = ts_node_start_byte(nd); int len = (int)(ts_node_end_byte(nd) - s); if (len > 0 && len < CBM_SZ_64 && s < (uint32_t)ctx->source_len) { @@ -128,6 +132,10 @@ static void compute_fingerprint(CBMExtractCtx *ctx, CBMDefinition *def, TSNode f if (ts_node_is_null(body)) { body = func_node; } + /* Extract raw identifier tokens from body for semantic search — before MinHash gate + * so short functions still get body_tokens even without a fingerprint. */ + def->body_tokens = extract_body_ident_tokens(ctx, body); + cbm_minhash_t result; if (!cbm_minhash_compute(body, ctx->source, (int)ctx->language, &result)) { return; /* Too short or empty — no fingerprint */ @@ -155,9 +163,6 @@ static void compute_fingerprint(CBMExtractCtx *ctx, CBMDefinition *def, TSNode f cbm_ast_profile_to_str(&profile, sp_buf, sizeof(sp_buf)); def->structural_profile = cbm_arena_strdup(ctx->arena, sp_buf); } - - /* Extract raw identifier tokens from body for semantic search */ - def->body_tokens = extract_body_ident_tokens(ctx, body); } // Tree-sitter row is 0-based; lines are 1-based. diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 686691974..293868e0b 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -802,6 +802,25 @@ TEST(ts_class) { PASS(); } +TEST(body_tokens_type_identifier) { + CBMFileResult *r = extract("function serialize(obj: MyModel): SerializedResult {\n" + " const result: SerializedResult = new SerializedResult();\n" + " return result;\n" + "}\n", + CBM_LANG_TYPESCRIPT, "t", "serial.ts"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].name, "serialize") == 0) { + ASSERT_NOT_NULL(r->defs.items[i].body_tokens); + ASSERT(strstr(r->defs.items[i].body_tokens, "SerializedResult") != NULL); + break; + } + } + cbm_free_result(r); + PASS(); +} + /* --- Lua --- */ TEST(lua_function) { CBMFileResult *r = extract( @@ -3683,6 +3702,7 @@ SUITE(extraction) { RUN_TEST(js_class); RUN_TEST(ts_function); RUN_TEST(ts_class); + RUN_TEST(body_tokens_type_identifier); RUN_TEST(lua_function); RUN_TEST(bash_function); RUN_TEST(perl_function);