feat: Add cube description to members for vector search indexing and agent output

packages/cubejs-api-gateway/openspec.yml

@@ -96,6 +96,9 @@ components:
           type: "string"
         description:
           type: "string"
+        cubeDescription:
+          type: "string"
+          description: "Description of the parent cube/view this segment belongs to"
         shortTitle:
           type: "string"
         meta:
@@ -132,6 +135,9 @@ components:
           type: "string"
         description:
           type: "string"
+        cubeDescription:
+          type: "string"
+          description: "Description of the parent cube/view this dimension belongs to"
         type:
           type: "string"
         aliasMember:
@@ -172,6 +178,9 @@ components:
           type: "string"
         description:
           type: "string"
+        cubeDescription:
+          type: "string"
+          description: "Description of the parent cube/view this measure belongs to"
         type:
           type: "string"
         aggType:

packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts

@@ -27,6 +27,7 @@ type ConfigItem = {
   title: string;
   shortTitle: string;
   description: string;
+  cubeDescription?: string;
   type: string;
   format: string;
   /** ISO 4217 currency code in uppercase (e.g. USD, EUR) */
@@ -61,6 +62,7 @@ const annotation = (
     title: config.title,
     shortTitle: config.shortTitle,
     description: config.description,
+    cubeDescription: config.cubeDescription,
     type: config.type,
     format: config.format,
     currency: config.currency,

packages/cubejs-api-gateway/test/index.test.ts

@@ -682,8 +682,11 @@ describe('API Gateway', () => {
     expect(res.body.cubes[0]?.description).toBe('cube from compilerApi mock');
     expect(res.body.cubes[0]?.hasOwnProperty('sql')).toBe(false);
     expect(res.body.cubes[0]?.dimensions.find(dimension => dimension.name === 'Foo.id').description).toBe('id dimension from compilerApi mock');
+    expect(res.body.cubes[0]?.dimensions.find(dimension => dimension.name === 'Foo.id').cubeDescription).toBe('cube from compilerApi mock');
     expect(res.body.cubes[0]?.measures.find(measure => measure.name === 'Foo.bar').description).toBe('measure from compilerApi mock');
+    expect(res.body.cubes[0]?.measures.find(measure => measure.name === 'Foo.bar').cubeDescription).toBe('cube from compilerApi mock');
     expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').description).toBe('segment from compilerApi mock');
+    expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').cubeDescription).toBe('cube from compilerApi mock');
   });
 
   test('meta endpoint extended to get schema information with additional data', async () => {
@@ -699,8 +702,11 @@ describe('API Gateway', () => {
     expect(res.body.cubes[0]?.description).toBe('cube from compilerApi mock');
     expect(res.body.cubes[0]?.hasOwnProperty('sql')).toBe(true);
     expect(res.body.cubes[0]?.dimensions.find(dimension => dimension.name === 'Foo.id').description).toBe('id dimension from compilerApi mock');
+    expect(res.body.cubes[0]?.dimensions.find(dimension => dimension.name === 'Foo.id').cubeDescription).toBe('cube from compilerApi mock');
     expect(res.body.cubes[0]?.measures.find(measure => measure.name === 'Foo.bar').description).toBe('measure from compilerApi mock');
+    expect(res.body.cubes[0]?.measures.find(measure => measure.name === 'Foo.bar').cubeDescription).toBe('cube from compilerApi mock');
     expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').description).toBe('segment from compilerApi mock');
+    expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').cubeDescription).toBe('cube from compilerApi mock');
   });
 
   describe('multi query support', () => {

packages/cubejs-api-gateway/test/mocks.ts

@@ -90,21 +90,25 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
             {
               name: 'Foo.bar',
               description: 'measure from compilerApi mock',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
           ],
           dimensions: [
             {
               name: 'Foo.id',
               description: 'id dimension from compilerApi mock',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
             {
               name: 'Foo.time',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
             {
               name: 'Foo.timeGranularities',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
               granularities: [
                 {
@@ -120,6 +124,7 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
             {
               name: 'Foo.quux',
               description: 'segment from compilerApi mock',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
           ],
@@ -138,6 +143,7 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
             {
               name: 'Foo.bar',
               description: 'measure from compilerApi mock',
+              cubeDescription: 'cube from compilerApi mock',
               sql: 'bar',
               isVisible: true,
             },
@@ -146,17 +152,20 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
             {
               name: 'Foo.id',
               description: 'id dimension from compilerApi mock',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
             {
               name: 'Foo.time',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
           ],
           segments: [
             {
               name: 'Foo.quux',
               description: 'segment from compilerApi mock',
+              cubeDescription: 'cube from compilerApi mock',
               isVisible: true,
             },
           ],

packages/cubejs-backend-shared/src/env.ts

@@ -298,6 +298,9 @@ const variables: Record<string, (...args: any) => any> = {
   nestedFoldersDelimiter: () => get('CUBEJS_NESTED_FOLDERS_DELIMITER')
     .default('')
     .asString(),
+  maxEmbeddingDescriptionLength: () => get('CUBEJS_MAX_EMBEDDING_DESCRIPTION_LENGTH')
+    .default('0')
+    .asInt(),
   defaultTimezone: () => get('CUBEJS_DEFAULT_TIMEZONE')
     .default('UTC')
     .asString(),

packages/cubejs-client-core/src/types.ts

@@ -377,6 +377,7 @@ export type BaseCubeMember = {
   title: string;
   shortTitle: string;
   description?: string;
+  cubeDescription?: string;
   /**
    * @deprecated use `public` instead
    */

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts

@@ -69,6 +69,7 @@ export type MeasureConfig = {
   name: string;
   title: string;
   description?: string;
+  cubeDescription?: string;
   shortTitle: string;
   format?: MeasureFormat;
   currency?: string;
@@ -92,6 +93,7 @@ export type DimensionConfig = {
   title: string;
   type: string;
   description?: string;
+  cubeDescription?: string;
   shortTitle: string;
   suggestFilterValues: boolean;
   format?: DimensionFormat;
@@ -111,6 +113,7 @@ export type SegmentConfig = {
   title: string;
   shortTitle: string;
   description?: string;
+  cubeDescription?: string;
   meta?: any;
   isVisible: boolean;
   public: boolean;
@@ -184,6 +187,17 @@ export class CubeToMetaTransformer implements CompilerInterface {
     this.queries = this.cubes;
   }
 
+  private static truncateDescription(description: string | undefined): string | undefined {
+    if (!description) {
+      return description;
+    }
+    const maxLen = getEnv('maxEmbeddingDescriptionLength');
+    if (maxLen > 0 && description.length > maxLen) {
+      return description.slice(0, maxLen);
+    }
+    return description;
+  }
+
   protected transform(cube: CubeDefinitionExtended, _errorReporter?: ErrorReporter): TransformedCube {
     const extendedCube = cube as ExtendedCubeDefinition;
     const cubeName = extendedCube.name;
@@ -229,21 +243,25 @@ export class CubeToMetaTransformer implements CompilerInterface {
 
     const nestedFolders: NestedFolder[] = (extendedCube.folders || []).map((f: Folder) => processFolder(f));
 
+    const truncDesc = CubeToMetaTransformer.truncateDescription;
+    const cubeDesc = truncDesc(extendedCube.description);
+
     return {
       config: {
         name: cubeName,
         type: extendedCube.isView ? 'view' : 'cube',
         title: cubeTitle,
         isVisible: isCubeVisible,
         public: isCubeVisible,
-        description: extendedCube.description,
+        description: cubeDesc,
         connectedComponent: this.joinGraph.connectedComponents()[cubeName],
         meta: extendedCube.meta,
         measures: Object.entries(extendedCube.measures || {}).map((nameToMetric: [string, any]) => {
           const metricDef = nameToMetric[1] as ExtendedCubeSymbolDefinition;
           const measureVisibility = isCubeVisible ? this.isVisible(metricDef, true) : false;
           return {
             ...this.measureConfig(cubeName, cubeTitle, nameToMetric),
+            cubeDescription: cubeDesc,
             isVisible: measureVisibility,
             public: measureVisibility,
           };
@@ -260,7 +278,8 @@ export class CubeToMetaTransformer implements CompilerInterface {
             name: `${cubeName}.${dimensionName}`,
             title: this.title(cubeTitle, nameToDimension, false),
             type: this.dimensionDataType(extendedDimDef.type || 'string'),
-            description: extendedDimDef.description,
+            description: truncDesc(extendedDimDef.description),
+            cubeDescription: cubeDesc,
             shortTitle: this.title(cubeTitle, nameToDimension, true),
             suggestFilterValues:
               extendedDimDef.suggestFilterValues == null
@@ -295,7 +314,8 @@ export class CubeToMetaTransformer implements CompilerInterface {
             name: `${cubeName}.${segmentName}`,
             title: this.title(cubeTitle, nameToSegment, false),
             shortTitle: this.title(cubeTitle, nameToSegment, true),
-            description: extendedSegmentDef.description,
+            description: truncDesc(extendedSegmentDef.description),
+            cubeDescription: cubeDesc,
             meta: extendedSegmentDef.meta,
             isVisible: segmentVisibility,
             public: segmentVisibility,
@@ -383,7 +403,7 @@ export class CubeToMetaTransformer implements CompilerInterface {
     return {
       name,
       title: this.title(cubeTitle, nameToMetric, false),
-      description: extendedMetricDef.description,
+      description: CubeToMetaTransformer.truncateDescription(extendedMetricDef.description),
       shortTitle: this.title(cubeTitle, nameToMetric, true),
       format: this.transformMeasureFormat(extendedMetricDef.format),
       currency: extendedMetricDef.currency?.toUpperCase(),

packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap

@@ -7,6 +7,7 @@ Object {
   "dimensions": Array [
     Object {
       "aliasMember": "CubeA.id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeA.id",
       "format": "imageUrl",
@@ -27,6 +28,7 @@ Object {
     },
     Object {
       "aliasMember": "CubeB.other_id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeB.other_id",
       "format": "imageUrl",
@@ -53,6 +55,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeA.count_a",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -76,6 +79,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeB.count_b",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -114,6 +118,7 @@ Object {
   "dimensions": Array [
     Object {
       "aliasMember": "CubeB.other_id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeB.other_id",
       "format": "imageUrl",
@@ -140,6 +145,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeA.count_a",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -163,6 +169,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeB.count_b",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -201,6 +208,7 @@ Object {
   "dimensions": Array [
     Object {
       "aliasMember": "CubeA.id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeA.id",
       "format": "imageUrl",
@@ -221,6 +229,7 @@ Object {
     },
     Object {
       "aliasMember": "CubeB.id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeB.id",
       "format": "imageUrl",
@@ -241,6 +250,7 @@ Object {
     },
     Object {
       "aliasMember": "CubeB.other_id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeB.other_id",
       "format": "imageUrl",
@@ -267,6 +277,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeA.count_a",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -290,6 +301,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeB.count_b",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -328,6 +340,7 @@ Object {
   "dimensions": Array [
     Object {
       "aliasMember": "CubeB.other_id",
+      "cubeDescription": undefined,
       "currency": undefined,
       "description": "Description for CubeB.other_id",
       "format": "imageUrl",
@@ -354,6 +367,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeA.count_a",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,
@@ -377,6 +391,7 @@ Object {
     Object {
       "aggType": "number",
       "aliasMember": "CubeB.count_b",
+      "cubeDescription": undefined,
       "cumulative": false,
       "cumulativeTotal": false,
       "currency": undefined,

rust/cubesql/cubeclient/src/models/v1_cube_meta_dimension.rs

@@ -21,6 +21,8 @@ pub struct V1CubeMetaDimension {
     pub short_title: Option<String>,
     #[serde(rename = "description", skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
+    #[serde(rename = "cubeDescription", skip_serializing_if = "Option::is_none")]
+    pub cube_description: Option<String>,
     #[serde(rename = "type")]
     pub r#type: String,
     /// When dimension is defined in View, it keeps the original path: Cube.dimension
@@ -48,6 +50,7 @@ impl V1CubeMetaDimension {
             title: None,
             short_title: None,
             description: None,
+            cube_description: None,
             r#type,
             alias_member: None,
             granularities: None,