Enh/ts 5255/cols func2

docs/en/14-reference/09-error-code.md

@@ -458,6 +458,10 @@ This document details the server error codes that may be encountered when using
 | 0x80002665 | The _TAGS pseudocolumn can only be used for subtable and supertable queries | Illegal tag column query                                     | Check and correct the SQL statement                          |
 | 0x80002666 | Subquery does not output primary timestamp column            | Check and correct the SQL statement                          |                                                              |
 | 0x80002667 | Invalid usage of expr: %s                                    | Illegal expression                                           | Check and correct the SQL statement                          |
+| 0x80002687 | Invalid using cols function                                  | Illegal using cols function                                        | Check and correct the SQL statement                          |
+| 0x80002688 | Cols function's first param must be a select function        | The first parameter of the cols function should be a selection function | Check and correct the SQL statement                          |
+| 0x80002689 | Invalid using cols function with multiple output columns     | Illegal using the cols function for multiple column output             | Check and correct the SQL statement                          |
+| 0x80002690 | Invalid using alias for cols function                        | Illegal cols function alias                                  | Check and correct the SQL statement                          |
 | 0x800026FF | Parser internal error                                        | Internal error in parser                                     | Preserve the scene and logs, report issue on GitHub          |
 | 0x80002700 | Planner internal error                                       | Internal error in planner                                    | Preserve the scene and logs, report issue on GitHub          |
 | 0x80002701 | Expect ts equal                                              | JOIN condition validation failed                             | Preserve the scene and logs, report issue on GitHub          |

docs/zh/14-reference/09-error-code.md

@@ -473,8 +473,12 @@ description: TDengine 服务端的错误码列表和详细说明
 | 0x80002663 | Not unique table/alias                                                                                 | 表名（别名）冲突                              | 检查并修正SQL语句                     |
 | 0x80002664 | Join requires valid time series input                                                                  | 不支持子查询不含主键时间戳列输出的JOIN查询    | 检查并修正SQL语句                     |
 | 0x80002665 | The _TAGS pseudo column can only be used for subtable and supertable queries                           | 非法TAG列查询                                 | 检查并修正SQL语句                     |
-| 0x80002666 | 子查询不含主键时间戳列输出                                                                             | 检查并修正SQL语句                             |
+| 0x80002666 | 子查询不含主键时间戳列输出                                                                                | 检查并修正SQL语句                             |
 | 0x80002667 | Invalid usage of expr: %s                                                                              | 非法表达式                                    | 检查并修正SQL语句                     |
+| 0x80002687 | Invalid using cols function                                                                            | cols函数使用错误                             | 检查并修正SQL语句                        |
+| 0x80002688 | Cols function's first param must be a select function                                                  | cols函数第一个参数应该为选择函数                | 检查并修正SQL语句                     |
+| 0x80002689 | Invalid using cols function with multiple output columns                                               | 多列输出的 cols 函数使用错误                   | 检查并修正SQL语句                      |
+| 0x80002690 | Invalid using alias for cols function                                                                  | cols 函数输出列重命名错误                      | 检查并修正SQL语句                     |
 | 0x800026FF | Parser internal error                                                                                  | 解析器内部错误                                | 保留现场和日志，github上报issue       |
 | 0x80002700 | Planner internal error                                                                                 | 计划期内部错误                                | 保留现场和日志，github上报issue       |
 | 0x80002701 | Expect ts equal                                                                                        | JOIN条件校验失败                              | 保留现场和日志，github上报issue       |

include/libs/function/function.h

@@ -282,6 +282,8 @@ typedef struct tExprNode {
       struct SNode *pRootNode;
     } _optrRoot;
   };
+  int32_t   bindTupleFuncIdx;
+  int32_t   tupleFuncIdx;
 } tExprNode;
 
 struct SScalarParam {

include/libs/function/functionMgt.h

@@ -155,6 +155,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_FORECAST_LOW,
   FUNCTION_TYPE_FORECAST_HIGH,
   FUNCTION_TYPE_FORECAST_ROWTS,
+  FUNCTION_TYPE_COLS,
   FUNCTION_TYPE_IROWTS_ORIGIN,
 
   // internal function
@@ -208,6 +209,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_HYPERLOGLOG_STATE,
   FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE,
 
+
   // geometry functions
   FUNCTION_TYPE_GEOM_FROM_TEXT = 4250,
   FUNCTION_TYPE_AS_TEXT,
@@ -295,6 +297,7 @@ bool fmisSelectGroupConstValueFunc(int32_t funcId);
 bool fmIsElapsedFunc(int32_t funcId);
 bool fmIsDBUsageFunc(int32_t funcId);
 bool fmIsRowTsOriginFunc(int32_t funcId);
+bool fmIsSelectColsFunc(int32_t funcId);
 
 void    getLastCacheDataType(SDataType* pType, int32_t pkBytes);
 int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);

include/libs/nodes/querynodes.h

@@ -16,6 +16,7 @@
 #ifndef _TD_QUERY_NODES_H_
 #define _TD_QUERY_NODES_H_
 
+#include <stdint.h>
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -61,6 +62,8 @@ typedef struct SExprNode {
   bool      asParam;
   bool      asPosition;
   int32_t   projIdx;
+  int32_t   bindTupleFuncIdx;
+  int32_t   tupleFuncIdx;
 } SExprNode;
 
 typedef enum EColumnType {

include/util/taoserror.h

@@ -906,6 +906,10 @@ int32_t  taosGetErrSize();
 #define TSDB_CODE_PAR_INVALID_ANOMALY_WIN_OPT   TAOS_DEF_ERROR_CODE(0, 0x2684)
 #define TSDB_CODE_PAR_INVALID_FORECAST_CLAUSE   TAOS_DEF_ERROR_CODE(0, 0x2685)
 #define TSDB_CODE_PAR_INVALID_VGID_LIST         TAOS_DEF_ERROR_CODE(0, 0x2686)
+#define TSDB_CODE_PAR_INVALID_COLS_FUNCTION     TAOS_DEF_ERROR_CODE(0, 0x2687)
+#define TSDB_CODE_PAR_INVALID_COLS_SELECTFUNC   TAOS_DEF_ERROR_CODE(0, 0x2688)
+#define TSDB_CODE_INVALID_MULITI_COLS_FUNC      TAOS_DEF_ERROR_CODE(0, 0x2689)
+#define TSDB_CODE_INVALID_COLS_ALIAS            TAOS_DEF_ERROR_CODE(0, 0x2690)
 #define TSDB_CODE_PAR_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x26FF)
 
 //planner

include/util/tdef.h

@@ -271,6 +271,7 @@ typedef enum ELogicConditionType {
 #define TSDB_SUBSCRIBE_KEY_LEN   (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
 #define TSDB_PARTITION_KEY_LEN   (TSDB_SUBSCRIBE_KEY_LEN + 20)
 #define TSDB_COL_NAME_LEN        65
+#define TSDB_COL_NAME_EXLEN      4
 #define TSDB_COL_FNAME_LEN       (TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
 #define TSDB_MAX_SAVED_SQL_LEN   TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN         TSDB_PAYLOAD_SIZE

source/libs/executor/src/aggregateoperator.c

@@ -395,6 +395,12 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
     return TSDB_CODE_SUCCESS;
   }
 
+  // if the last expression is a tuple function, we don't need to create a empty data block
+  int32_t lastExprIndex = pOperator->exprSupp.numOfExprs - 1;
+  if(pOperator->exprSupp.pExprInfo[lastExprIndex].pExpr->tupleFuncIdx > 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   code = createDataBlock(&pBlock);
   if (code) {
     return code;

source/libs/executor/src/executil.c

@@ -18,6 +18,8 @@
 #include "index.h"
 #include "os.h"
 #include "query.h"
+#include "querynodes.h"
+#include "tarray.h"
 #include "tdatablock.h"
 #include "thash.h"
 #include "tmsg.h"
@@ -1990,7 +1992,8 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
     code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
     QUERY_CHECK_CODE(code, lino, _end);
   }
-
+  pExp->pExpr->bindTupleFuncIdx = ((SExprNode*)pNode)->bindTupleFuncIdx;
+  pExp->pExpr->tupleFuncIdx = ((SExprNode*)pNode)->tupleFuncIdx;
 _end:
   if (code != TSDB_CODE_SUCCESS) {
     qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
@@ -2071,42 +2074,81 @@ int32_t createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, SExprInfo**
   return code;
 }
 
+static void deleteSubsidiareCtx(void* pData) {
+  SSubsidiaryResInfo* pCtx = (SSubsidiaryResInfo*)pData;
+  if (pCtx->pCtx) {
+    taosMemoryFreeClear(pCtx->pCtx);
+  }
+}
+
 // set the output buffer for the selectivity + tag query
 static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   int32_t num = 0;
   int32_t code = TSDB_CODE_SUCCESS;
   int32_t lino = 0;
 
-  SqlFunctionCtx*  p = NULL;
-  SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES);
-  if (pValCtx == NULL) {
-    return terrno;
+  SArray* pValCtxArray = NULL;
+  for (int32_t i = numOfOutput - 1; i > 0; --i) {  // select Func is at the end of the list
+    int32_t funcIdx = pCtx[i].pExpr->pExpr->tupleFuncIdx;
+    if (funcIdx > 0) {
+      if (pValCtxArray == NULL) {
+        // the end of the list is the select function of biggest index
+        pValCtxArray = taosArrayInit_s(sizeof(SSubsidiaryResInfo*), funcIdx);
+        if (pValCtxArray == NULL) {
+          return terrno;
+        }
+      }
+      if (funcIdx > pValCtxArray->size) {
+        qError("funcIdx:%d is out of range", funcIdx);
+        taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx);
+        return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
+      }
+      SSubsidiaryResInfo* pSubsidiary = &pCtx[i].subsidiaries;
+      pSubsidiary->pCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES);
+      if (pSubsidiary->pCtx == NULL) {
+        taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx);
+        return terrno;
+      }
+      pSubsidiary->num = 0;
+      taosArraySet(pValCtxArray, funcIdx - 1, &pSubsidiary);
+    } else {
+      break;
+    }
   }
 
-  SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
-  QUERY_CHECK_NULL(pSelectFuncs, code, lino, _end, terrno);
+  SqlFunctionCtx*  p = NULL;
+  SqlFunctionCtx** pValCtx = NULL;
+  if (pValCtxArray == NULL) {
+    pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES);
+    if (pValCtx == NULL) {
+      QUERY_CHECK_CODE(terrno, lino, _end);
+    }
+  }
 
   for (int32_t i = 0; i < numOfOutput; ++i) {
     const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
     if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0) ||
         (strcmp(pName, "_group_const_value") == 0)) {
-      pValCtx[num++] = &pCtx[i];
-    } else if (fmIsSelectFunc(pCtx[i].functionId)) {
-      void* data = taosHashGet(pSelectFuncs, pName, strlen(pName));
-      if (taosHashGetSize(pSelectFuncs) != 0 && data == NULL) {
-        p = NULL;
-        break;
+      if (pValCtxArray == NULL) {
+        pValCtx[num++] = &pCtx[i];
       } else {
-        int32_t tempRes = taosHashPut(pSelectFuncs, pName, strlen(pName), &num, sizeof(num));
-        if (tempRes != TSDB_CODE_SUCCESS && tempRes != TSDB_CODE_DUP_KEY) {
-          code = tempRes;
-          QUERY_CHECK_CODE(code, lino, _end);
+        int32_t bindFuncIndex = pCtx[i].pExpr->pExpr->bindTupleFuncIdx;  // start from index 1;
+        if (bindFuncIndex > 0) {  // 0 is default index related to the select function
+          bindFuncIndex -= 1;
         }
-        p = &pCtx[i];
+        SSubsidiaryResInfo** pSubsidiary = taosArrayGet(pValCtxArray, bindFuncIndex);
+        if(pSubsidiary == NULL) {
+          QUERY_CHECK_CODE(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR, lino, _end);
+        }
+        (*pSubsidiary)->pCtx[(*pSubsidiary)->num] = &pCtx[i];
+        (*pSubsidiary)->num++;
       }
+    } else if (fmIsSelectFunc(pCtx[i].functionId)) {
+       if (pValCtxArray == NULL) {
+        p = &pCtx[i];
+       }
     }
   }
-  taosHashCleanup(pSelectFuncs);
 
   if (p != NULL) {
     p->subsidiaries.pCtx = pValCtx;
@@ -2117,9 +2159,11 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
 
 _end:
   if (code != TSDB_CODE_SUCCESS) {
+    taosArrayDestroyP(pValCtxArray, deleteSubsidiareCtx);
     taosMemoryFreeClear(pValCtx);
-    taosHashCleanup(pSelectFuncs);
     qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+  } else {
+    taosArrayDestroy(pValCtxArray);
   }
   return code;
 }

source/libs/function/inc/functionMgtInt.h

@@ -59,6 +59,7 @@ extern "C" {
 #define FUNC_MGT_COUNT_LIKE_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found
 #define FUNC_MGT_PROCESS_BY_ROW         FUNC_MGT_FUNC_CLASSIFICATION_MASK(31)
 #define FUNC_MGT_FORECAST_PC_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(32)
+#define FUNC_MGT_SELECT_COLS_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(33)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 

source/libs/function/src/builtins.c

@@ -1696,6 +1696,10 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateColsFunction(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return TSDB_CODE_PAR_INVALID_COLS_FUNCTION;
+}
+
 static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len));
   int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
@@ -2731,7 +2735,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .translateFunc = translateOutFirstIn,
     .dynDataRequiredFunc = firstDynDataReq,
     .getEnvFunc   = getFirstLastFuncEnv,
-    .initFunc     = functionSetup,
+    .initFunc     = firstLastFunctionSetup,
     .processFunc  = firstFunction,
     .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize,
@@ -5641,7 +5645,11 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
                    .paramInfoPattern = 0,
                    .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_VARCHAR_TYPE}},
     .translateFunc = translateOutVarchar,
-  }
+  },
+  {
+    .name = "cols",
+    .translateFunc = translateColsFunction,
+  },
 };
 // clang-format on
 

source/libs/function/src/builtinsimpl.c

@@ -946,7 +946,7 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu
       if (nullList[j]) {
         colDataSetNULL(pDstCol, rowIndex);
       } else {
-        code = colDataSetVal(pDstCol, rowIndex, pStart, false);
+        code = colDataSetValOrCover(pDstCol, rowIndex, pStart, false);
         if (TSDB_CODE_SUCCESS != code) {
           return code;
         }
@@ -2403,7 +2403,6 @@ int32_t firstLastFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResIn
   }
 
   SFirstLastRes*        pRes = GET_ROWCELL_INTERBUF(pResInfo);
-  SInputColumnInfoData* pInput = &pCtx->input;
 
   pRes->nullTupleSaved = false;
   pRes->nullTuplePos.pageId = -1;
@@ -2740,7 +2739,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
 
       if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
         char*   data = colDataGetData(pInputCol, chosen);
-        int32_t code = doSaveCurrentVal(pCtx, i, cts, NULL, type, data);
+        int32_t code = doSaveCurrentVal(pCtx, chosen, cts, NULL, type, data);
         if (code != TSDB_CODE_SUCCESS) {
           return code;
         }

source/libs/function/src/functionMgt.c

@@ -173,6 +173,8 @@ bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC
 
 bool fmIsVectorFunc(int32_t funcId) { return !fmIsScalarFunc(funcId) && !fmIsPseudoColumnFunc(funcId); }
 
+bool fmIsSelectColsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_COLS_FUNC); }
+
 bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); }
 
 bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); }
@@ -441,6 +443,8 @@ int32_t createFunctionWithSrcFunc(const char* pName, const SFunctionNode* pSrcFu
     return code;
   }
   resetOutputChangedFunc(*ppFunc, pSrcFunc);
+  (*ppFunc)->node.bindTupleFuncIdx = pSrcFunc->node.bindTupleFuncIdx;
+  (*ppFunc)->node.tupleFuncIdx = pSrcFunc->node.tupleFuncIdx;
   return code;
 }
 

source/libs/nodes/src/nodesCloneFuncs.c

@@ -103,6 +103,8 @@ static int32_t exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
   COPY_CHAR_ARRAY_FIELD(aliasName);
   COPY_CHAR_ARRAY_FIELD(userAlias);
   COPY_SCALAR_FIELD(projIdx);
+  COPY_SCALAR_FIELD(bindTupleFuncIdx);
+  COPY_SCALAR_FIELD(tupleFuncIdx);
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/nodes/src/nodesEqualFuncs.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "functionMgt.h"
 #include "querynodes.h"
 
 #define COMPARE_SCALAR_FIELD(fldname)           \
@@ -137,6 +138,11 @@ static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) {
   COMPARE_SCALAR_FIELD(funcId);
   COMPARE_STRING_FIELD(functionName);
   COMPARE_NODE_LIST_FIELD(pParameterList);
+  if (a->funcType == FUNCTION_TYPE_SELECT_VALUE) {
+    if ((a->node.bindTupleFuncIdx != b->node.bindTupleFuncIdx) &&
+        (a->node.bindTupleFuncIdx != 0 || b->node.bindTupleFuncIdx != 0))
+      return false;
+  }
   return true;
 }