func/regexp_extract: new scalar func and test cases

include/libs/function/functionMgt.h

@@ -141,6 +141,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_AES_DECRYPT,
   FUNCTION_TYPE_SM4_ENCRYPT,
   FUNCTION_TYPE_SM4_DECRYPT,
+  FUNCTION_TYPE_REGEXP_EXTRACT,
 
   // conversion function
   FUNCTION_TYPE_CAST = 2000,

include/libs/scalar/scalar.h

@@ -128,6 +128,7 @@ int32_t crc32Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
 int32_t findInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t likeInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t regexpInSetFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t regexpExtractFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t generateTotpSecretFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t generateTotpCodeFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 

source/libs/function/src/builtins.c

@@ -1102,13 +1102,58 @@ static int32_t translateRand(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return TSDB_CODE_SUCCESS;
 }
 
-// return type is same as first input parameter's type
-static int32_t translateOutFirstIn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+static int32_t translateRegexpExtract(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+
+  // param[1]: pattern must be a constant VALUE node
+  SNode* pPatNode = nodesListGetNode(pFunc->pParameterList, 1);
+  if (QUERY_NODE_VALUE != nodeType(pPatNode)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, "regexp_extract: pattern must be a constant");
+  }
+
+  // Validate the regex pattern compiles as POSIX ERE
+  SValueNode* pPatVal = (SValueNode*)pPatNode;
+  if (pPatVal->literal != NULL) {
+    regex_t re;
+    int     ret = regcomp(&re, pPatVal->literal, REG_EXTENDED);
+    if (ret != 0) {
+      char msgbuf[256] = {0};
+      (void)regerror(ret, &re, msgbuf, sizeof(msgbuf));
+      regfree(&re);
+      return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR,
+                             "Invalid regex pattern for regexp_extract: %s", msgbuf);
+    }
+    regfree(&re);
+  }
+
+  // param[2]: group_idx (optional) must be a non-negative integer constant
+  if (numOfParams == 3) {
+    SNode* pIdxNode = nodesListGetNode(pFunc->pParameterList, 2);
+    if (QUERY_NODE_VALUE != nodeType(pIdxNode)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, "regexp_extract: group_idx must be a constant integer");
+    }
+    SValueNode* pIdxVal = (SValueNode*)pIdxNode;
+    if (!IS_INTEGER_TYPE(pIdxVal->node.resType.type)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, "regexp_extract: group_idx must be an integer");
+    }
+    int64_t groupIdx = taosStr2Int64(pIdxVal->literal, NULL, 10);
+    if (groupIdx < 0 || groupIdx > 512) {
+      return invaildFuncParaValueErrMsg(pErrBuf, len, "regexp_extract: group_idx must be between 0 and 512");
+    }
+  }
+
+  // Return type matches str (param[0]): same VARCHAR/NCHAR type and byte width
   pFunc->node.resType = *getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0));
   return TSDB_CODE_SUCCESS;
 }
 
+// return type is same as first input parameter's type
+static int32_t translateOutFirstIn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));  pFunc->node.resType = *getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0));
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translatePlaceHolderPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // pseudo column do not need to check parameters
   switch (pFunc->funcType) {
@@ -7414,6 +7459,41 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .sprocessFunc = streamPseudoScalarFunction,
     .finalizeFunc = NULL,
   },
+  {
+    .name = "regexp_extract",
+    .type = FUNCTION_TYPE_REGEXP_EXTRACT,
+    .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
+    .parameters = {.minParamNum = 2,
+                   .maxParamNum = 3,
+                   .paramInfoPattern = 1,
+                   .inputParaInfo[0][0] = {.isLastParam = false,
+                                           .startParam = 1,
+                                           .endParam = 1,
+                                           .validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE | FUNC_PARAM_SUPPORT_NCHAR_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE,
+                                           .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE,
+                                           .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE,
+                                           .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,},
+                   .inputParaInfo[0][1] = {.isLastParam = false,
+                                           .startParam = 2,
+                                           .endParam = 2,
+                                           .validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE | FUNC_PARAM_SUPPORT_NCHAR_TYPE,
+                                           .validNodeType = FUNC_PARAM_SUPPORT_VALUE_NODE,
+                                           .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE,
+                                           .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,},
+                   .inputParaInfo[0][2] = {.isLastParam = true,
+                                           .startParam = 3,
+                                           .endParam = 3,
+                                           .validDataType = FUNC_PARAM_SUPPORT_INTEGER_TYPE,
+                                           .validNodeType = FUNC_PARAM_SUPPORT_VALUE_NODE,
+                                           .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE,
+                                           .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,},
+                   .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE | FUNC_PARAM_SUPPORT_NCHAR_TYPE}},
+    .translateFunc = translateRegexpExtract,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = regexpExtractFunction,
+    .finalizeFunc = NULL,
+  },
 };
 // clang-format on
 

source/libs/scalar/src/sclfunc.c

@@ -1817,6 +1817,175 @@ static int32_t base32Encode(const uint8_t *in, int32_t inLen, char *out) {
   return outLen;
 }
 
+int32_t regexpExtractFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  int32_t          numOfRows  = pInput[0].numOfRows;
+  SColumnInfoData *pStrData   = pInput[0].columnData;
+  SColumnInfoData *pPatData   = pInput[1].columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  if (numOfRows == 0) {
+    pOutput->numOfRows = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // NULL-type str: all output rows are NULL
+  if (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))) {
+    colDataSetNNULL(pOutputData, 0, numOfRows);
+    pOutput->numOfRows = numOfRows;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // NULL pattern: all output rows are NULL
+  if (colDataIsNull_s(pPatData, 0)) {
+    colDataSetNNULL(pOutputData, 0, numOfRows);
+    pOutput->numOfRows = numOfRows;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // Get group_idx (default 1; param[2] is an optional integer constant)
+  int32_t groupIdx = 1;
+  if (inputNum == 3 && !IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[2])) && !colDataIsNull_s(pInput[2].columnData, 0)) {
+    GET_TYPED_DATA(groupIdx, int32_t, GET_PARAM_TYPE(&pInput[2]),
+                   colDataGetData(pInput[2].columnData, 0),
+                   typeGetTypeModFromColInfo(&pInput[2].columnData->info));
+  }
+  if (groupIdx < 0 || groupIdx > 512) {
+    return TSDB_CODE_FUNC_FUNTION_PARA_VALUE;
+  }
+
+  // Build null-terminated UTF-8 pattern string (pattern is a constant, always 1 row)
+  char    patBuf[512];
+  char   *patStr     = patBuf;
+  int32_t patLen     = 0;
+  bool    needFreePat = false;
+  {
+    char   *rawPat    = varDataVal(colDataGetData(pPatData, 0));
+    int32_t rawPatLen = varDataLen(colDataGetData(pPatData, 0));
+    if (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) {
+      SCL_ERR_RET(convNcharToVarchar(rawPat, &patStr, rawPatLen, &patLen, pInput[1].charsetCxt));
+      needFreePat = true;
+    } else {
+      patLen = rawPatLen;
+      if (patLen >= (int32_t)sizeof(patBuf)) {
+        patStr = taosMemoryMalloc(patLen + 1);
+        if (patStr == NULL) return terrno;
+        needFreePat = true;
+      }
+      (void)memcpy(patStr, rawPat, patLen);
+      patStr[patLen] = '\0';
+    }
+  }
+
+  // Compile (or retrieve cached) regex — pattern is constant so cache hits every row
+  regex_t *regex = NULL;
+  if (threadGetRegComp(&regex, patStr) != 0) {
+    code = TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR;
+    goto _exit;
+  }
+
+  // regmatch_t array: index 0 = whole match, 1..groupIdx = capture groups
+  int32_t     nmatch  = groupIdx + 1;
+  regmatch_t *pmatch  = taosMemoryMalloc(nmatch * sizeof(regmatch_t));
+  if (pmatch == NULL) {
+    code = terrno;
+    goto _exit;
+  }
+
+  // Output buffer: same byte width as the str column
+  int32_t outBufLen = pStrData->info.bytes;
+  char   *outBuf    = taosMemoryMalloc(outBufLen);
+  if (outBuf == NULL) {
+    taosMemoryFree(pmatch);
+    code = terrno;
+    goto _exit;
+  }
+
+  int32_t strType = GET_PARAM_TYPE(&pInput[0]);
+  bool    isNchar = (strType == TSDB_DATA_TYPE_NCHAR);
+
+  for (int32_t i = 0; i < numOfRows; i++) {
+    if (colDataIsNull_s(pStrData, i)) {
+      colDataSetNULL(pOutputData, i);
+      continue;
+    }
+
+    char   *strRaw = colDataGetData(pStrData, i);
+    char   *strVal = varDataVal(strRaw);
+    int32_t strLen = varDataLen(strRaw);
+
+    // For NCHAR (UCS-4), convert to UTF-8 before matching
+    char   *strUtf8     = strVal;
+    int32_t strUtf8Len  = strLen;
+    bool    needFreeUtf8 = false;
+    if (isNchar) {
+      if (convNcharToVarchar(strVal, &strUtf8, strLen, &strUtf8Len, pInput[0].charsetCxt) != 0) {
+        colDataSetNULL(pOutputData, i);
+        continue;
+      }
+      needFreeUtf8 = true;
+    }
+
+    // Null-terminate the string for regexec
+    char  ntBuf[1024];
+    char *strNt      = ntBuf;
+    bool  needFreeNt = false;
+    if (strUtf8Len >= (int32_t)sizeof(ntBuf)) {
+      strNt      = taosMemoryMalloc(strUtf8Len + 1);
+      needFreeNt = true;
+      if (strNt == NULL) {
+        if (needFreeUtf8) taosMemoryFree(strUtf8);
+        colDataSetNULL(pOutputData, i);
+        continue;
+      }
+    }
+    (void)memcpy(strNt, strUtf8, strUtf8Len);
+    strNt[strUtf8Len] = '\0';
+
+    int ret = regexec(regex, strNt, nmatch, pmatch, 0);
+    if (ret != 0 || pmatch[groupIdx].rm_so == -1) {
+      // REG_NOMATCH, or the requested capture group did not participate
+      colDataSetNULL(pOutputData, i);
+    } else {
+      int32_t matchStart = pmatch[groupIdx].rm_so;
+      int32_t matchLen   = pmatch[groupIdx].rm_eo - pmatch[groupIdx].rm_so;
+
+      if (isNchar) {
+        // Convert matched UTF-8 bytes back to NCHAR (UCS-4)
+        char   *matchedNchar    = NULL;
+        int32_t matchedNcharLen = 0;
+        if (convVarcharToNchar(strNt + matchStart, &matchedNchar, matchLen, &matchedNcharLen,
+                               pInput[0].charsetCxt) != 0) {
+          colDataSetNULL(pOutputData, i);
+        } else {
+          *(VarDataLenT *)outBuf = matchedNcharLen;
+          (void)memcpy(outBuf + VARSTR_HEADER_SIZE, matchedNchar, matchedNcharLen);
+          taosMemoryFree(matchedNchar);
+          code = colDataSetVal(pOutputData, i, outBuf, false);
+          if (code != TSDB_CODE_SUCCESS) terrno = code;
+        }
+      } else {
+        *(VarDataLenT *)outBuf = matchLen;
+        (void)memcpy(outBuf + VARSTR_HEADER_SIZE, strNt + matchStart, matchLen);
+        code = colDataSetVal(pOutputData, i, outBuf, false);
+        if (code != TSDB_CODE_SUCCESS) terrno = code;
+      }
+    }
+
+    if (needFreeNt)   taosMemoryFree(strNt);
+    if (needFreeUtf8) taosMemoryFree(strUtf8);
+    if (code != TSDB_CODE_SUCCESS) break;
+  }
+
+  taosMemoryFree(outBuf);
+  taosMemoryFree(pmatch);
+_exit:
+  if (needFreePat) taosMemoryFree(patStr);
+  pOutput->numOfRows = numOfRows;
+  return code;
+}
+
 int32_t generateTotpSecretFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   SColumnInfoData *pInputData = pInput->columnData;
   SColumnInfoData *pOutputData = pOutput->columnData;