diff --git a/ui/src/components/DataSourceFormModal.tsx b/ui/src/components/DataSourceFormModal.tsx index d9fa822d381..328c614d3d2 100644 --- a/ui/src/components/DataSourceFormModal.tsx +++ b/ui/src/components/DataSourceFormModal.tsx @@ -7,11 +7,16 @@ import { EuiHorizontalRule, EuiText, EuiCallOut, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiTextArea, + EuiTitle, } from "@elastic/eui"; import { feast } from "../protos"; import FormModal from "./forms/FormModal"; import TagsEditor, { TagEntry } from "./forms/TagsEditor"; -import NameDescriptionOwnerFields from "./forms/NameDescriptionOwnerFields"; +import { DATA_SOURCE_TYPES } from "../pages/data-sources/DataSourceCatalog"; const SOURCE_TYPE_OPTIONS = [ { @@ -30,13 +35,25 @@ const SOURCE_TYPE_OPTIONS = [ value: String(feast.core.DataSource.SourceType.BATCH_REDSHIFT), text: "Redshift", }, + { + value: String(feast.core.DataSource.SourceType.BATCH_SPARK), + text: "Spark", + }, + { + value: String(feast.core.DataSource.SourceType.BATCH_TRINO), + text: "Trino", + }, + { + value: String(feast.core.DataSource.SourceType.BATCH_ATHENA), + text: "AWS Athena", + }, { value: String(feast.core.DataSource.SourceType.STREAM_KAFKA), text: "Kafka", }, { - value: String(feast.core.DataSource.SourceType.BATCH_SPARK), - text: "Spark", + value: String(feast.core.DataSource.SourceType.STREAM_KINESIS), + text: "AWS Kinesis", }, { value: String(feast.core.DataSource.SourceType.REQUEST_SOURCE), @@ -46,6 +63,17 @@ const SOURCE_TYPE_OPTIONS = [ value: String(feast.core.DataSource.SourceType.PUSH_SOURCE), text: "Push Source", }, + { + value: String(feast.core.DataSource.SourceType.CUSTOM_SOURCE), + text: "Custom Source", + }, + { value: "RAY_SOURCE", text: "Ray" }, + { value: "POSTGRES_SOURCE", text: "PostgreSQL" }, + { value: "MONGODB_SOURCE", text: "MongoDB" }, + { value: "CLICKHOUSE_SOURCE", text: "ClickHouse" }, + { value: "MSSQL_SOURCE", text: "SQL Server" }, + { value: "ORACLE_SOURCE", text: "Oracle" }, + { value: "COUCHBASE_SOURCE", text: "Couchbase" }, ]; interface DataSourceFormData { @@ -69,6 +97,33 @@ interface DataSourceFormData { kafkaTopic: string; sparkTable: string; sparkPath: string; + kinesisRegion: string; + kinesisStreamName: string; + trinoTable: string; + trinoQuery: string; + athenaTable: string; + athenaQuery: string; + athenaDatabase: string; + athenaDataSource: string; + customSourceClassName: string; + customSourceConfig: string; + // Contrib source fields + rayReaderType: string; + rayPath: string; + rayReaderOptions: string; + postgresTable: string; + postgresQuery: string; + mongodbCollection: string; + clickhouseTable: string; + clickhouseQuery: string; + mssqlTable: string; + mssqlConnectionStr: string; + oracleTable: string; + oracleConnectionStr: string; + couchbaseDatabase: string; + couchbaseScope: string; + couchbaseCollection: string; + couchbaseQuery: string; } interface DataSourceFormModalProps { @@ -101,6 +156,32 @@ const EMPTY_FORM: DataSourceFormData = { kafkaTopic: "", sparkTable: "", sparkPath: "", + kinesisRegion: "", + kinesisStreamName: "", + trinoTable: "", + trinoQuery: "", + athenaTable: "", + athenaQuery: "", + athenaDatabase: "", + athenaDataSource: "", + customSourceClassName: "", + customSourceConfig: "", + rayReaderType: "parquet", + rayPath: "", + rayReaderOptions: "", + postgresTable: "", + postgresQuery: "", + mongodbCollection: "", + clickhouseTable: "", + clickhouseQuery: "", + mssqlTable: "", + mssqlConnectionStr: "", + oracleTable: "", + oracleConnectionStr: "", + couchbaseDatabase: "", + couchbaseScope: "", + couchbaseCollection: "", + couchbaseQuery: "", }; const BATCH_SOURCE_TYPES = new Set([ @@ -109,8 +190,31 @@ const BATCH_SOURCE_TYPES = new Set([ String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE), String(feast.core.DataSource.SourceType.BATCH_REDSHIFT), String(feast.core.DataSource.SourceType.BATCH_SPARK), + String(feast.core.DataSource.SourceType.BATCH_TRINO), + String(feast.core.DataSource.SourceType.BATCH_ATHENA), + "RAY_SOURCE", + "POSTGRES_SOURCE", + "MONGODB_SOURCE", + "CLICKHOUSE_SOURCE", + "MSSQL_SOURCE", + "ORACLE_SOURCE", + "COUCHBASE_SOURCE", ]); +const RAY_READER_OPTIONS = [ + { value: "parquet", text: "Parquet" }, + { value: "csv", text: "CSV" }, + { value: "json", text: "JSON" }, + { value: "text", text: "Text" }, + { value: "images", text: "Images" }, + { value: "binary_files", text: "Binary Files" }, + { value: "tfrecords", text: "TFRecords" }, + { value: "webdataset", text: "WebDataset" }, + { value: "huggingface", text: "HuggingFace" }, + { value: "mongo", text: "MongoDB (via Ray)" }, + { value: "sql", text: "SQL (via Ray)" }, +]; + const DataSourceFormModal: React.FC = ({ onClose, onSubmit, @@ -125,13 +229,12 @@ const DataSourceFormModal: React.FC = ({ const [errors, setErrors] = useState>({}); const [submitted, setSubmitted] = useState(false); - useEffect(() => { - if (initialData) { - setFormData(initialData); - } - }, [initialData]); - const isBatchSource = BATCH_SOURCE_TYPES.has(formData.sourceType); + const isPreselected = !!initialData?.sourceType; + + const catalogEntry = DATA_SOURCE_TYPES.find( + (ds) => ds.sourceType === formData.sourceType, + ); const validate = (): boolean => { const newErrors: Record = {}; @@ -144,7 +247,6 @@ const DataSourceFormModal: React.FC = ({ "Must start with a letter or underscore, and contain only letters, numbers, and underscores."; } - // Source-type-specific required fields if (st === String(feast.core.DataSource.SourceType.BATCH_FILE)) { if (!formData.fileUri.trim()) { newErrors.fileUri = "File URI is required."; @@ -180,6 +282,19 @@ const DataSourceFormModal: React.FC = ({ newErrors.sparkTable = "Either a table reference or a path is required for Spark."; } + } else if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { + if (!formData.trinoTable.trim() && !formData.trinoQuery.trim()) { + newErrors.trinoTable = + "Either a table reference or a query is required for Trino."; + } + } else if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { + if (!formData.athenaTable.trim() && !formData.athenaQuery.trim()) { + newErrors.athenaTable = + "Either a table reference or a query is required for Athena."; + } + if (!formData.athenaDatabase.trim()) { + newErrors.athenaDatabase = "Database is required for Athena."; + } } else if (st === String(feast.core.DataSource.SourceType.STREAM_KAFKA)) { if (!formData.kafkaBootstrapServers.trim()) { newErrors.kafkaBootstrapServers = "Bootstrap servers are required."; @@ -194,12 +309,56 @@ const DataSourceFormModal: React.FC = ({ if (!formData.kafkaTopic.trim()) { newErrors.kafkaTopic = "Topic is required."; } + } else if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { + if (!formData.kinesisRegion.trim()) { + newErrors.kinesisRegion = "AWS region is required."; + } + if (!formData.kinesisStreamName.trim()) { + newErrors.kinesisStreamName = "Stream name is required."; + } + } else if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { + if (!formData.customSourceClassName.trim()) { + newErrors.customSourceClassName = "Class name is required."; + } + } else if (st === "RAY_SOURCE") { + if ( + !formData.rayPath.trim() && + !["huggingface", "mongo", "sql"].includes(formData.rayReaderType) + ) { + newErrors.rayPath = "Path is required for this reader type."; + } + } else if (st === "POSTGRES_SOURCE") { + if (!formData.postgresTable.trim() && !formData.postgresQuery.trim()) { + newErrors.postgresTable = "Either a table or query is required."; + } + } else if (st === "CLICKHOUSE_SOURCE") { + if ( + !formData.clickhouseTable.trim() && + !formData.clickhouseQuery.trim() + ) { + newErrors.clickhouseTable = "Either a table or query is required."; + } + } else if (st === "MSSQL_SOURCE") { + if (!formData.mssqlTable.trim()) { + newErrors.mssqlTable = "Table reference is required."; + } + } else if (st === "ORACLE_SOURCE") { + if (!formData.oracleTable.trim()) { + newErrors.oracleTable = "Table reference is required."; + } + } else if (st === "COUCHBASE_SOURCE") { + if ( + !formData.couchbaseCollection.trim() && + !formData.couchbaseQuery.trim() + ) { + newErrors.couchbaseCollection = + "Either a collection or query is required."; + } } - // Timestamp field required for batch sources (needed for point-in-time correctness) if (isBatchSource && !formData.timestampField.trim()) { newErrors.timestampField = - "Timestamp field is required for batch sources. It is used for point-in-time correct feature retrieval."; + "Timestamp field is required for batch sources."; } else if ( formData.timestampField.trim() && !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(formData.timestampField.trim()) @@ -242,25 +401,65 @@ const DataSourceFormModal: React.FC = ({ } }; + const renderFileSourceFields = () => ( + + updateField("fileUri", e.target.value)} + isInvalid={!!errors.fileUri} + placeholder="s3://bucket/path/to/data.parquet" + /> + + ); + + const renderSourceTypeHeader = () => { + if (!isPreselected || !catalogEntry) return null; + + const IconComponent = catalogEntry.icon; + return ( + + + +
+ +
+
+ + + {catalogEntry.name} + + + {catalogEntry.description} + + +
+
+ ); + }; + const renderSourceTypeFields = () => { const st = formData.sourceType; if (st === String(feast.core.DataSource.SourceType.BATCH_FILE)) { - return ( - - updateField("fileUri", e.target.value)} - isInvalid={!!errors.fileUri} - placeholder="s3://bucket/path/to/data.parquet" - /> - - ); + return renderFileSourceFields(); } if (st === String(feast.core.DataSource.SourceType.BATCH_BIGQUERY)) { @@ -280,13 +479,14 @@ const DataSourceFormModal: React.FC = ({ /> - updateField("bigqueryQuery", e.target.value)} - placeholder="SELECT * FROM `project.dataset.table`" + placeholder="SELECT * FROM `project.dataset.table` WHERE ..." + rows={3} /> @@ -296,25 +496,35 @@ const DataSourceFormModal: React.FC = ({ if (st === String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE)) { return ( <> - - updateField("snowflakeDatabase", e.target.value)} - isInvalid={!!errors.snowflakeDatabase} - placeholder="MY_DATABASE" - /> - - - updateField("snowflakeSchema", e.target.value)} - placeholder="PUBLIC" - /> - + + + + + updateField("snowflakeDatabase", e.target.value) + } + isInvalid={!!errors.snowflakeDatabase} + placeholder="MY_DATABASE" + /> + + + + + + updateField("snowflakeSchema", e.target.value) + } + placeholder="PUBLIC" + /> + + + = ({ if (st === String(feast.core.DataSource.SourceType.BATCH_REDSHIFT)) { return ( <> - - updateField("redshiftDatabase", e.target.value)} - isInvalid={!!errors.redshiftDatabase} - placeholder="my_database" - /> - - - updateField("redshiftSchema", e.target.value)} - placeholder="public" - /> - + + + + + updateField("redshiftDatabase", e.target.value) + } + isInvalid={!!errors.redshiftDatabase} + placeholder="my_database" + /> + + + + + + updateField("redshiftSchema", e.target.value) + } + placeholder="public" + /> + + + = ({ label="Bootstrap Servers" isInvalid={!!errors.kafkaBootstrapServers} error={errors.kafkaBootstrapServers} - helpText="Comma-separated list of broker host:port pairs." + helpText="Comma-separated host:port pairs." > = ({ label="Table" isInvalid={!!errors.sparkTable} error={errors.sparkTable} - helpText="Spark catalog table reference (catalog.database.table). Provide either table or path." + helpText="Spark catalog table (catalog.database.table). Provide either table or path." > = ({ /> = ({ ); } + if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { + return ( + <> + + updateField("trinoTable", e.target.value)} + isInvalid={!!errors.trinoTable} + placeholder="catalog.schema.table" + /> + + + updateField("trinoQuery", e.target.value)} + placeholder="SELECT * FROM catalog.schema.table" + rows={3} + /> + + + ); + } + + if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { + return ( + <> + + + + + updateField("athenaDatabase", e.target.value) + } + isInvalid={!!errors.athenaDatabase} + placeholder="my_database" + /> + + + + + + updateField("athenaDataSource", e.target.value) + } + placeholder="AwsDataCatalog" + /> + + + + + updateField("athenaTable", e.target.value)} + isInvalid={!!errors.athenaTable} + placeholder="my_table" + /> + + + updateField("athenaQuery", e.target.value)} + placeholder="SELECT * FROM my_table" + rows={3} + /> + + + ); + } + + if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { + return ( + <> + + updateField("kinesisRegion", e.target.value)} + isInvalid={!!errors.kinesisRegion} + placeholder="us-east-1" + /> + + + updateField("kinesisStreamName", e.target.value)} + isInvalid={!!errors.kinesisStreamName} + placeholder="my-feature-stream" + /> + + + ); + } + + if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { + return ( + <> + + + updateField("customSourceClassName", e.target.value) + } + isInvalid={!!errors.customSourceClassName} + placeholder="mymodule.MyCustomDataSource" + /> + + + + updateField("customSourceConfig", e.target.value) + } + placeholder='{"key": "value"}' + rows={3} + /> + + + ); + } + + if (st === "RAY_SOURCE") { + return ( + <> + + updateField("rayReaderType", e.target.value)} + /> + + + updateField("rayPath", e.target.value)} + isInvalid={!!errors.rayPath} + placeholder="s3://bucket/images/" + /> + + + updateField("rayReaderOptions", e.target.value)} + placeholder='{"dataset_name": "org/name", "split": "train"}' + rows={3} + /> + + + ); + } + + if (st === "POSTGRES_SOURCE") { + return ( + <> + + updateField("postgresTable", e.target.value)} + isInvalid={!!errors.postgresTable} + placeholder="public.my_features" + /> + + + updateField("postgresQuery", e.target.value)} + placeholder="SELECT * FROM my_features WHERE ..." + rows={3} + /> + + + ); + } + + if (st === "MONGODB_SOURCE") { + return ( + + updateField("mongodbCollection", e.target.value)} + isInvalid={!!errors.mongodbCollection} + placeholder="features_collection" + /> + + ); + } + + if (st === "CLICKHOUSE_SOURCE") { + return ( + <> + + updateField("clickhouseTable", e.target.value)} + isInvalid={!!errors.clickhouseTable} + placeholder="default.my_features" + /> + + + updateField("clickhouseQuery", e.target.value)} + placeholder="SELECT * FROM default.my_features" + rows={3} + /> + + + ); + } + + if (st === "MSSQL_SOURCE") { + return ( + <> + + updateField("mssqlTable", e.target.value)} + isInvalid={!!errors.mssqlTable} + placeholder="dbo.my_features" + /> + + + + updateField("mssqlConnectionStr", e.target.value) + } + placeholder="mssql+pyodbc://user:pass@host/db" // pragma: allowlist secret + /> + + + ); + } + + if (st === "ORACLE_SOURCE") { + return ( + <> + + updateField("oracleTable", e.target.value)} + isInvalid={!!errors.oracleTable} + placeholder="SCHEMA.MY_FEATURES" + /> + + + + updateField("oracleConnectionStr", e.target.value) + } + placeholder="oracle+cx_oracle://user:pass@host:1521/service" // pragma: allowlist secret + /> + + + ); + } + + if (st === "COUCHBASE_SOURCE") { + return ( + <> + + + + + updateField("couchbaseDatabase", e.target.value) + } + placeholder="Default" + /> + + + + + + updateField("couchbaseScope", e.target.value) + } + placeholder="Default" + /> + + + + + + updateField("couchbaseCollection", e.target.value) + } + isInvalid={!!errors.couchbaseCollection} + placeholder="my_collection" + /> + + + updateField("couchbaseQuery", e.target.value)} + placeholder="SELECT * FROM `collection`" + rows={3} + /> + + + ); + } + if ( st === String(feast.core.DataSource.SourceType.REQUEST_SOURCE) || st === String(feast.core.DataSource.SourceType.PUSH_SOURCE) ) { return ( - - No additional configuration required for this source type. - + + + No connection configuration needed. This source type receives data + at request time or via push ingestion. + + ); } return null; }; + const sourceTypeName = + SOURCE_TYPE_OPTIONS.find((o) => o.value === formData.sourceType)?.text || + "Data Source"; + return ( {submitError && ( <> = ({ )} - updateField("name", v)} - onChangeDescription={(v) => updateField("description", v)} - onChangeOwner={(v) => updateField("owner", v)} - nameDisabled={isEdit} - nameError={errors.name} - nameHelpText="A unique name for this data source." - namePlaceholder="e.g. customer_transactions" - descriptionPlaceholder="Describe this data source..." - /> + {isPreselected && renderSourceTypeHeader()} + {isPreselected && } - - { - updateField("sourceType", e.target.value); - // Clear source-specific errors when type changes - setErrors((prev) => { - const next = { ...prev }; - delete next.fileUri; - delete next.bigqueryTable; - delete next.snowflakeTable; - delete next.snowflakeDatabase; - delete next.redshiftTable; - delete next.redshiftDatabase; - delete next.kafkaBootstrapServers; - delete next.kafkaTopic; - delete next.sparkTable; - delete next.timestampField; - return next; - }); - }} - disabled={isEdit} + {/* Section: Identity */} + +

Identity

+
+ + + + + + updateField("name", e.target.value)} + isInvalid={!!errors.name} + disabled={isEdit} + placeholder="e.g. customer_transactions" + /> + + + + + updateField("owner", e.target.value)} + placeholder="team@company.com" + /> + + + + + + updateField("description", e.target.value)} + placeholder="Brief description of this data source..." /> + {!isPreselected && ( + <> + + + { + updateField("sourceType", e.target.value); + setErrors({}); + }} + disabled={isEdit} + /> + + + )} + - -

Source Configuration

-
+ + {/* Section: Connection */} + +

Connection Details

+
{renderSourceTypeFields()} + {/* Section: Timestamp (for batch sources) */} {isBatchSource && ( <> - - updateField("timestampField", e.target.value)} - isInvalid={!!errors.timestampField} - placeholder="event_timestamp" - /> - - - - updateField("createdTimestampColumn", e.target.value) - } - placeholder="created_at" - /> - + + +

Time Configuration

+
+ + + + + + + updateField("timestampField", e.target.value) + } + isInvalid={!!errors.timestampField} + placeholder="event_timestamp" + /> + + + + + + updateField("createdTimestampColumn", e.target.value) + } + placeholder="created_at" + /> + + + )} + {/* Section: Tags */} + + +

Tags (optional)

+
+ = ({ table: dsData.sparkTable, path: dsData.sparkPath, }; + } else if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { + payload.trino_options = { + table: dsData.trinoTable, + query: dsData.trinoQuery, + }; + } else if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { + payload.athena_options = { + table: dsData.athenaTable, + query: dsData.athenaQuery, + database: dsData.athenaDatabase, + data_source: dsData.athenaDataSource, + }; + } else if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { + payload.kinesis_options = { + region: dsData.kinesisRegion, + stream_name: dsData.kinesisStreamName, + }; + } else if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { + payload.custom_options = { + class_name: dsData.customSourceClassName, + config: dsData.customSourceConfig, + }; } applyDataSource.mutate(payload as any, { diff --git a/ui/src/graphics/data-source-icons.tsx b/ui/src/graphics/data-source-icons.tsx new file mode 100644 index 00000000000..813c714a560 --- /dev/null +++ b/ui/src/graphics/data-source-icons.tsx @@ -0,0 +1,377 @@ +import React from "react"; + +export const BigQueryIcon = (props: React.SVGProps) => ( + + + + + + +); + +export const SnowflakeIcon = (props: React.SVGProps) => ( + + + + +); + +export const RedshiftIcon = (props: React.SVGProps) => ( + + + + + + +); + +export const KafkaIcon = (props: React.SVGProps) => ( + + + + + + + + + + + + + + + +); + +export const SparkIcon = (props: React.SVGProps) => ( + + + + + +); + +export const FileIcon = (props: React.SVGProps) => ( + + + + + + + + +); + +export const RequestSourceIcon = (props: React.SVGProps) => ( + + + + + + + +); + +export const PushSourceIcon = (props: React.SVGProps) => ( + + + + + +); + +export const KinesisIcon = (props: React.SVGProps) => ( + + + + + +); + +export const TrinoIcon = (props: React.SVGProps) => ( + + + + + + + +); + +export const AthenaIcon = (props: React.SVGProps) => ( + + + + +); + +export const CustomSourceIcon = (props: React.SVGProps) => ( + + + + +); + +export const RayIcon = (props: React.SVGProps) => ( + + + + + +); + +export const PostgresIcon = (props: React.SVGProps) => ( + + + + + + +); + +export const MongoDBIcon = (props: React.SVGProps) => ( + + + + + +); + +export const SqlServerIcon = (props: React.SVGProps) => ( + + + + + + +); + +export const OracleIcon = (props: React.SVGProps) => ( + + + + + ORA + + +); + +export const CouchbaseIcon = (props: React.SVGProps) => ( + + + + + + +); + +export const ClickHouseIcon = (props: React.SVGProps) => ( + + + + + + + +); diff --git a/ui/src/pages/data-sources/DataSourceCatalog.tsx b/ui/src/pages/data-sources/DataSourceCatalog.tsx new file mode 100644 index 00000000000..fef72186bb0 --- /dev/null +++ b/ui/src/pages/data-sources/DataSourceCatalog.tsx @@ -0,0 +1,369 @@ +import React, { useState } from "react"; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiText, + EuiTitle, + EuiSpacer, + EuiButton, + EuiBadge, +} from "@elastic/eui"; +import { feast } from "../../protos"; +import { + BigQueryIcon, + SnowflakeIcon, + RedshiftIcon, + KafkaIcon, + SparkIcon, + FileIcon, + RequestSourceIcon, + PushSourceIcon, + KinesisIcon, + TrinoIcon, + AthenaIcon, + CustomSourceIcon, + RayIcon, + PostgresIcon, + MongoDBIcon, + SqlServerIcon, + OracleIcon, + CouchbaseIcon, + ClickHouseIcon, +} from "../../graphics/data-source-icons"; + +interface DataSourceTypeInfo { + id: string; + sourceType: string; + name: string; + description: string; + icon: React.FC>; + category: "batch" | "stream" | "on-demand"; + color: string; + contrib?: boolean; +} + +const DATA_SOURCE_TYPES: DataSourceTypeInfo[] = [ + { + id: "bigquery", + sourceType: String(feast.core.DataSource.SourceType.BATCH_BIGQUERY), + name: "BigQuery", + description: + "Google Cloud's serverless data warehouse. Ideal for large-scale analytics and ML feature computation.", + icon: BigQueryIcon, + category: "batch", + color: "#4285F4", + }, + { + id: "snowflake", + sourceType: String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE), + name: "Snowflake", + description: + "Cloud-native data platform with elastic scaling. Connect to your Snowflake tables for feature engineering.", + icon: SnowflakeIcon, + category: "batch", + color: "#29B5E8", + }, + { + id: "redshift", + sourceType: String(feast.core.DataSource.SourceType.BATCH_REDSHIFT), + name: "Redshift", + description: + "AWS fully managed data warehouse. Pull features from your Redshift clusters with fast parallel queries.", + icon: RedshiftIcon, + category: "batch", + color: "#205B97", + }, + { + id: "spark", + sourceType: String(feast.core.DataSource.SourceType.BATCH_SPARK), + name: "Spark", + description: + "Apache Spark data source for distributed processing. Access tables via Spark catalog or direct file paths.", + icon: SparkIcon, + category: "batch", + color: "#E25A1C", + }, + { + id: "file", + sourceType: String(feast.core.DataSource.SourceType.BATCH_FILE), + name: "File (Parquet / CSV)", + description: + "Read features from Parquet or CSV files stored in S3, GCS, HDFS, or local filesystem.", + icon: FileIcon, + category: "batch", + color: "#4CAF50", + }, + { + id: "trino", + sourceType: String(feast.core.DataSource.SourceType.BATCH_TRINO), + name: "Trino", + description: + "Distributed SQL query engine for big data analytics. Query data across heterogeneous sources via Trino catalog.", + icon: TrinoIcon, + category: "batch", + color: "#DD00A1", + }, + { + id: "athena", + sourceType: String(feast.core.DataSource.SourceType.BATCH_ATHENA), + name: "AWS Athena", + description: + "Serverless interactive query service on AWS. Run SQL queries directly against data in S3 without infrastructure.", + icon: AthenaIcon, + category: "batch", + color: "#8C4FFF", + }, + { + id: "kafka", + sourceType: String(feast.core.DataSource.SourceType.STREAM_KAFKA), + name: "Kafka", + description: + "Real-time event streaming platform. Ingest features from Kafka topics for low-latency serving.", + icon: KafkaIcon, + category: "stream", + color: "#231F20", + }, + { + id: "kinesis", + sourceType: String(feast.core.DataSource.SourceType.STREAM_KINESIS), + name: "AWS Kinesis", + description: + "Managed real-time data streaming on AWS. Capture and process streaming data at scale for real-time features.", + icon: KinesisIcon, + category: "stream", + color: "#FF9900", + }, + { + id: "request-source", + sourceType: String(feast.core.DataSource.SourceType.REQUEST_SOURCE), + name: "Request Source", + description: + "Features provided at request time by the caller. No external storage needed — values come from the client.", + icon: RequestSourceIcon, + category: "on-demand", + color: "#7B61FF", + }, + { + id: "push-source", + sourceType: String(feast.core.DataSource.SourceType.PUSH_SOURCE), + name: "Push Source", + description: + "Push-based ingestion source. Clients push feature values directly to the online/offline store.", + icon: PushSourceIcon, + category: "on-demand", + color: "#FF6B35", + }, + { + id: "ray", + sourceType: "RAY_SOURCE", + name: "Ray", + description: + "Multi-format data source powered by Ray. Read images, HuggingFace datasets, Parquet, CSV, MongoDB, and more via Ray Data.", + icon: RayIcon, + category: "batch", + color: "#00A2E8", + contrib: true, + }, + { + id: "postgres", + sourceType: "POSTGRES_SOURCE", + name: "PostgreSQL", + description: + "Open-source relational database. Query feature data from PostgreSQL tables with full SQL support.", + icon: PostgresIcon, + category: "batch", + color: "#336791", + contrib: true, + }, + { + id: "mongodb", + sourceType: "MONGODB_SOURCE", + name: "MongoDB", + description: + "Document-oriented NoSQL database. Access feature data stored in MongoDB collections.", + icon: MongoDBIcon, + category: "batch", + color: "#00684A", + contrib: true, + }, + { + id: "clickhouse", + sourceType: "CLICKHOUSE_SOURCE", + name: "ClickHouse", + description: + "Column-oriented OLAP database for real-time analytics. High-performance queries for feature retrieval.", + icon: ClickHouseIcon, + category: "batch", + color: "#FFCC00", + contrib: true, + }, + { + id: "mssql", + sourceType: "MSSQL_SOURCE", + name: "SQL Server", + description: + "Microsoft SQL Server data source. Connect to MSSQL databases for enterprise feature data.", + icon: SqlServerIcon, + category: "batch", + color: "#CC2927", + contrib: true, + }, + { + id: "oracle", + sourceType: "ORACLE_SOURCE", + name: "Oracle", + description: + "Oracle Database data source. Pull features from Oracle tables and views for enterprise workloads.", + icon: OracleIcon, + category: "batch", + color: "#F80000", + contrib: true, + }, + { + id: "couchbase", + sourceType: "COUCHBASE_SOURCE", + name: "Couchbase", + description: + "Couchbase Columnar analytics source. Run SQL++ queries across distributed data in Couchbase.", + icon: CouchbaseIcon, + category: "batch", + color: "#EA2328", + contrib: true, + }, + { + id: "custom-source", + sourceType: String(feast.core.DataSource.SourceType.CUSTOM_SOURCE), + name: "Custom Source", + description: + "Plugin-based data source for custom integrations. Extend Feast with your own data source implementation.", + icon: CustomSourceIcon, + category: "batch", + color: "#607D8B", + }, +]; + +const CATEGORY_LABELS: Record = { + batch: { label: "Batch", color: "primary" }, + stream: { label: "Streaming", color: "accent" }, + "on-demand": { label: "On-Demand", color: "warning" }, +}; + +interface DataSourceCatalogProps { + onSelectType: (sourceType: string) => void; +} + +const SourceCard: React.FC<{ + dsType: DataSourceTypeInfo; + isHovered: boolean; + onHover: (id: string | null) => void; + onSelect: (sourceType: string) => void; +}> = ({ dsType, isHovered, onHover, onSelect }) => { + const categoryInfo = CATEGORY_LABELS[dsType.category]; + + return ( + + onHover(dsType.id)} + onMouseLeave={() => onHover(null)} + style={{ + height: "100%", + display: "flex", + flexDirection: "column", + transition: "all 0.2s ease", + transform: isHovered ? "translateY(-2px)" : "none", + borderTop: `3px solid ${dsType.color}`, + cursor: "pointer", + }} + onClick={() => onSelect(dsType.sourceType)} + > + + +
+ +
+
+ + +

{dsType.name}

+
+
+ + + {categoryInfo.label} + + +
+ + + + +

{dsType.description}

+
+ + + + { + e.stopPropagation(); + onSelect(dsType.sourceType); + }} + iconType="plusInCircle" + size="s" + > + Create Connection + +
+
+ ); +}; + +const DataSourceCatalog: React.FC = ({ + onSelectType, +}) => { + const [hoveredId, setHoveredId] = useState(null); + + return ( +
+ +

+ Choose a data source type to create a new connection. Each type has + its own configuration tailored to the underlying storage system. +

+
+ + + + {DATA_SOURCE_TYPES.map((dsType) => ( + + ))} + +
+ ); +}; + +export default DataSourceCatalog; +export { DATA_SOURCE_TYPES }; +export type { DataSourceTypeInfo }; diff --git a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx index 6ee71f9a65c..42a8d2536f0 100644 --- a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx +++ b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx @@ -69,6 +69,38 @@ const buildEditFormData = (ds: any): DataSourceFormData => { kafkaTopic: spec.kafkaOptions?.topic || ds.kafkaOptions?.topic || "", sparkTable: spec.sparkOptions?.table || ds.sparkOptions?.table || "", sparkPath: spec.sparkOptions?.path || ds.sparkOptions?.path || "", + kinesisRegion: + spec.kinesisOptions?.region || ds.kinesisOptions?.region || "", + kinesisStreamName: + spec.kinesisOptions?.streamName || ds.kinesisOptions?.streamName || "", + trinoTable: spec.trinoOptions?.table || ds.trinoOptions?.table || "", + trinoQuery: spec.trinoOptions?.query || ds.trinoOptions?.query || "", + athenaTable: spec.athenaOptions?.table || ds.athenaOptions?.table || "", + athenaQuery: spec.athenaOptions?.query || ds.athenaOptions?.query || "", + athenaDatabase: + spec.athenaOptions?.database || ds.athenaOptions?.database || "", + athenaDataSource: + spec.athenaOptions?.dataSource || ds.athenaOptions?.dataSource || "", + customSourceClassName: + spec.customOptions?.className || ds.customOptions?.className || "", + customSourceConfig: + spec.customOptions?.config || ds.customOptions?.config || "", + rayReaderType: "", + rayPath: "", + rayReaderOptions: "", + postgresTable: "", + postgresQuery: "", + mongodbCollection: "", + clickhouseTable: "", + clickhouseQuery: "", + mssqlTable: "", + mssqlConnectionStr: "", + oracleTable: "", + oracleConnectionStr: "", + couchbaseDatabase: "", + couchbaseScope: "", + couchbaseCollection: "", + couchbaseQuery: "", }; }; @@ -116,6 +148,28 @@ const formDataToPayload = (formData: DataSourceFormData, project: string) => { table: formData.sparkTable, path: formData.sparkPath, }; + } else if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { + payload.trino_options = { + table: formData.trinoTable, + query: formData.trinoQuery, + }; + } else if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { + payload.athena_options = { + table: formData.athenaTable, + query: formData.athenaQuery, + database: formData.athenaDatabase, + data_source: formData.athenaDataSource, + }; + } else if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { + payload.kinesis_options = { + region: formData.kinesisRegion, + stream_name: formData.kinesisStreamName, + }; + } else if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { + payload.custom_options = { + class_name: formData.customSourceClassName, + config: formData.customSourceConfig, + }; } return payload; diff --git a/ui/src/pages/data-sources/Index.tsx b/ui/src/pages/data-sources/Index.tsx index cf44de1770f..0a6a0d7148c 100644 --- a/ui/src/pages/data-sources/Index.tsx +++ b/ui/src/pages/data-sources/Index.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useMemo, useState } from "react"; import { useParams } from "react-router-dom"; import { @@ -14,8 +14,8 @@ import { } from "@elastic/eui"; import DatasourcesListingTable from "./DataSourcesListingTable"; +import DataSourceCatalog from "./DataSourceCatalog"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; -import DataSourceIndexEmptyState from "./DataSourceIndexEmptyState"; import { DataSourceIcon } from "../../graphics/DataSourceIcon"; import { useSearchQuery } from "../../hooks/useSearchInputWithTags"; import { feast } from "../../protos"; @@ -95,6 +95,28 @@ const formDataToPayload = (formData: DataSourceFormData, project: string) => { table: formData.sparkTable, path: formData.sparkPath, }; + } else if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { + payload.trino_options = { + table: formData.trinoTable, + query: formData.trinoQuery, + }; + } else if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { + payload.athena_options = { + table: formData.athenaTable, + query: formData.athenaQuery, + database: formData.athenaDatabase, + data_source: formData.athenaDataSource, + }; + } else if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { + payload.kinesis_options = { + region: formData.kinesisRegion, + stream_name: formData.kinesisStreamName, + }; + } else if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { + payload.custom_options = { + class_name: formData.customSourceClassName, + config: formData.customSourceConfig, + }; } return payload; @@ -105,7 +127,11 @@ const Index = () => { const { isLoading, isSuccess, isError, data } = useLoadDatasources(); const isAllProjects = projectName === "all"; + const [showCatalog, setShowCatalog] = useState(false); const [isModalOpen, setIsModalOpen] = useState(false); + const [preselectedSourceType, setPreselectedSourceType] = useState< + string | null + >(null); const [successMessage, setSuccessMessage] = useState(null); const [errorMessage, setErrorMessage] = useState(null); const applyDataSource = useApplyDataSource(); @@ -116,11 +142,73 @@ const Index = () => { const filterResult = data ? filterFn(data, searchTokens) : data; + const hasExistingSources = isSuccess && data && data.length > 0; + const isEmpty = isSuccess && (!data || data.length === 0); + + const modalInitialData = useMemo(() => { + if (!preselectedSourceType) return undefined; + return { + name: "", + description: "", + owner: "", + sourceType: preselectedSourceType, + timestampField: "", + createdTimestampColumn: "", + tags: [] as { key: string; value: string }[], + fileUri: "", + bigqueryTable: "", + bigqueryQuery: "", + snowflakeTable: "", + snowflakeDatabase: "", + snowflakeSchema: "", + redshiftTable: "", + redshiftDatabase: "", + redshiftSchema: "", + kafkaBootstrapServers: "", + kafkaTopic: "", + sparkTable: "", + sparkPath: "", + kinesisRegion: "", + kinesisStreamName: "", + trinoTable: "", + trinoQuery: "", + athenaTable: "", + athenaQuery: "", + athenaDatabase: "", + athenaDataSource: "", + customSourceClassName: "", + customSourceConfig: "", + rayReaderType: "parquet", + rayPath: "", + rayReaderOptions: "", + postgresTable: "", + postgresQuery: "", + mongodbCollection: "", + clickhouseTable: "", + clickhouseQuery: "", + mssqlTable: "", + mssqlConnectionStr: "", + oracleTable: "", + oracleConnectionStr: "", + couchbaseDatabase: "", + couchbaseScope: "", + couchbaseCollection: "", + couchbaseQuery: "", + }; + }, [preselectedSourceType]); + + const handleSelectType = (sourceType: string) => { + setPreselectedSourceType(sourceType); + setIsModalOpen(true); + }; + const handleCreateSubmit = (formData: DataSourceFormData) => { const payload = formDataToPayload(formData, projectName || ""); applyDataSource.mutate(payload as any, { onSuccess: () => { setIsModalOpen(false); + setPreselectedSourceType(null); + setShowCatalog(false); setErrorMessage(null); setSuccessMessage( `Data source "${formData.name}" created successfully.`, @@ -128,7 +216,6 @@ const Index = () => { setTimeout(() => setSuccessMessage(null), 5000); }, onError: (err: unknown) => { - // Error shown inside the modal via submitError prop const message = err instanceof Error ? err.message : "An unexpected error occurred."; setErrorMessage(message); @@ -143,13 +230,13 @@ const Index = () => { iconType={DataSourceIcon} pageTitle="Data Sources" rightSideItems={[ - ...(isAllProjects + ...(isAllProjects || showCatalog ? [] : [ setIsModalOpen(true)} + onClick={() => setShowCatalog(true)} key="create" > Create Data Source @@ -175,7 +262,7 @@ const Index = () => { )} - {errorMessage && ( + {errorMessage && !isModalOpen && ( <> { )} - {isLoading && ( -

- Loading -

- )} - {isError &&

We encountered an error while loading.

} - {isSuccess && !data && } - {isSuccess && data && data.length > 0 && filterResult && ( - - - - -

Search

+ + {showCatalog && !isAllProjects && ( + <> + + + +

Select a Data Source Type

- { - setSearchString(e.target.value); - }} - />
+ {hasExistingSources && ( + + setShowCatalog(false)} + iconType="arrowLeft" + size="s" + > + Back to Data Sources + + + )}
- -
+ + + )} + + {!showCatalog && ( + <> + {isLoading && ( +

+ Loading +

+ )} + {isError &&

We encountered an error while loading.

} + {isEmpty && !isAllProjects && ( + <> + +

No data sources yet — create your first connection

+
+ + + + )} + {isEmpty && isAllProjects && ( +

No data sources found across projects.

+ )} + {hasExistingSources && filterResult && ( + + + + +

Search

+
+ { + setSearchString(e.target.value); + }} + /> +
+
+ + +
+ )} + )} @@ -219,11 +347,13 @@ const Index = () => { { setIsModalOpen(false); + setPreselectedSourceType(null); setErrorMessage(null); }} onSubmit={handleCreateSubmit} isSubmitting={applyDataSource.isLoading} submitError={errorMessage} + initialData={modalInitialData} /> )}