Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ jobs:
env:
MXBAI_API_KEY: ${{ secrets.MXBAI_API_KEY }}
MXBAI_VECTOR_STORE_ID: ${{ secrets.MXBAI_VECTOR_STORE_ID }}
run: pnpm mxbai vs sync "${MXBAI_VECTOR_STORE_ID}" "src/content/docs/docs/**/*.mdx" --yes --strategy fast
run: pnpm mxbai store sync "${MXBAI_VECTOR_STORE_ID}" "src/content/docs/docs/**/*.mdx" --yes --strategy fast
2 changes: 1 addition & 1 deletion content/astro.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ export default defineConfig({
}),
starlightMixedbread({
apiKey: process.env.MXBAI_API_KEY ?? "",
vectorStoreId: process.env.MXBAI_VECTOR_STORE_ID ?? "",
storeId: process.env.MXBAI_VECTOR_STORE_ID ?? "",
maxResults: 8,
}),
effectPlaygroundPlugin({
Expand Down
4 changes: 2 additions & 2 deletions content/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@
"@expressive-code/plugin-collapsible-sections": "^0.40.2",
"@expressive-code/plugin-line-numbers": "^0.40.2",
"@faker-js/faker": "^9.3.0",
"@mixedbread/cli": "^1.0.1",
"@mixedbread/sdk": "^0.19.0",
"@mixedbread/cli": "^2.3.2",
"@mixedbread/sdk": "^0.61.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
"@opentelemetry/sdk-trace-base": "^1.28.0",
"@radix-ui/react-accordion": "^1.2.1",
Expand Down
707 changes: 217 additions & 490 deletions content/pnpm-lock.yaml

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions content/scripts/mixedbread.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ const cwd = process.cwd()
const path = Path.join(cwd, "src/content/docs/docs/**/*.mdx")
const filePaths = await Glob(path)

const vectorStoreId = process.env.MXBAI_VECTOR_STORE_ID!
const storeId = process.env.MXBAI_VECTOR_STORE_ID!
for (const filePath of filePaths) {
const file = await toFile(
Fs.createReadStream(filePath),
Path.basename(filePath),
{ type: "text/markdown" }
)
const url = filePath.replace(Path.join(cwd, "src/content/docs"), "")
await mxbai.vectorStores.files.upload(vectorStoreId, file, {
await mxbai.stores.files.upload(storeId, file, {
metadata: {
urlPath: url.split(".")[0],
}
Expand Down
10 changes: 5 additions & 5 deletions content/src/plugins/starlight/mixedbread/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ const __dirname = fileURLToPath(new URL(".", import.meta.url));
export interface MixedbreadClientOptions {
/** Your Mixedbread API key */
apiKey: string;
/** Your Vector Store ID */
vectorStoreId: string;
/** Your Store ID */
storeId: string;
/** The maximum number of results to return per search */
maxResults?: number;
/** Base URL for the Mixedbread API */
Expand All @@ -25,8 +25,8 @@ const MixedbreadConfigSchema = z
// Required config without which Mixedbread won"t work.
/** Your Mixedbread API key. */
apiKey: z.string(),
/** Your Vector Store ID. */
vectorStoreId: z.string(),
/** Your Store ID. */
storeId: z.string(),
// Optional config
/**
* The maximum number of results to return per search.
Expand Down Expand Up @@ -73,7 +73,7 @@ const MixedbreadConfigSchema = z
*
* export default {
* apiKey: process.env.MXBAI_API_KEY,
* vectorStoreId: process.env.VECTOR_STORE_ID,
* storeId: process.env.VECTOR_STORE_ID,
* maxResults: 15,
* } satisfies MixedbreadClientOptions;
*/
Expand Down
2 changes: 1 addition & 1 deletion content/src/plugins/starlight/mixedbread/virtual.d.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
declare module "virtual:starlight/mixedbread-config" {
const config: {
apiKey: string;
vectorStoreId: string;
storeId: string;
maxResults?: number;
baseUrl?: string;
disableUserPersonalization?: boolean;
Expand Down
8 changes: 4 additions & 4 deletions content/src/services/search/domain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ export const ScoredTextInputChunk = Schema.Struct({
Schema.propertySignature,
Schema.fromKey("file_id")
),
vectorStoreId: Schema.String.pipe(
storeId: Schema.String.pipe(
Schema.propertySignature,
Schema.fromKey("vector_store_id")
Schema.fromKey("store_id")
),
chunkIndex: Schema.Int.pipe(
Schema.propertySignature,
Expand All @@ -101,12 +101,12 @@ export const ScoredTextInputChunk = Schema.Struct({

export type ScoredTextInputChunk = typeof ScoredTextInputChunk.Type

export const VectorStoreSearchResponse = Schema.Struct({
export const StoreSearchResponse = Schema.Struct({
object: Schema.Literal("list"),
data: Schema.Array(ScoredTextInputChunk)
})

export type VectorStoreSearchResponse = typeof VectorStoreSearchResponse.Type
export type StoreSearchResponse = typeof StoreSearchResponse.Type

export const SearchResultChunk = Schema.Struct({
id: Schema.String,
Expand Down
14 changes: 7 additions & 7 deletions content/src/services/search/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,18 @@ import * as Layer from "effect/Layer"
import * as Redacted from "effect/Redacted"
import * as Predicate from "effect/Predicate"
import * as Schema from "effect/Schema"
import { SearchError, VectorStoreSearchResponse } from "./domain"
import { SearchError, StoreSearchResponse } from "./domain"
import type { Metadata, SearchResult } from "./domain"
import type { DeepMutable } from "effect/Types"

export class Search extends Effect.Service<Search>()("app/Search", {
effect: Effect.gen(function* () {
const apiKey = yield* Config.redacted("MXBAI_API_KEY")
const vectorStoreId = yield* Config.string("MXBAI_VECTOR_STORE_ID")
const storeId = yield* Config.string("MXBAI_VECTOR_STORE_ID")
const path = yield* Path.Path
const mxbai = new Mixedbread({ apiKey: Redacted.value(apiKey) })

const decodeSearchResponse = Schema.decodeUnknown(VectorStoreSearchResponse)
const decodeSearchResponse = Schema.decodeUnknown(StoreSearchResponse)

function extractSnippet(text: string, maxLength: number = 150): string {
// Remove markdown and clean up the text
Expand Down Expand Up @@ -77,7 +77,7 @@ export class Search extends Effect.Service<Search>()("app/Search", {
return path.join(parsed.dir, parsed.name, "/")
}

function groupSearchResults(response: VectorStoreSearchResponse): ReadonlyArray<SearchResult> {
function groupSearchResults(response: StoreSearchResponse): ReadonlyArray<SearchResult> {
const grouped = new Map<string, DeepMutable<SearchResult>>()

response.data.forEach((chunk) => {
Expand Down Expand Up @@ -127,15 +127,15 @@ export class Search extends Effect.Service<Search>()("app/Search", {
}

const search = Effect.fn("Search.search")(function* (query: string) {
const searchParams: Mixedbread.VectorStores.VectorStoreSearchParams = {
const searchParams: Mixedbread.Stores.StoreSearchParams = {
query,
top_k: 10,
search_options: { rerank: true, return_metadata: true },
vector_store_identifiers: [vectorStoreId],
store_identifiers: [storeId],
}

const response = yield* Effect.tryPromise({
try: (signal) => mxbai.vectorStores.search(searchParams, { signal }),
try: (signal) => mxbai.stores.search(searchParams, { signal }),
catch: (cause) => new SearchError({ cause }),
}).pipe(Effect.flatMap(decodeSearchResponse))

Expand Down