mirror of
https://github.com/Kaelio/ktx.git
synced 2026-07-22 11:51:01 +02:00
Add gdrive context source adapter (#209)
* Add gdrive context source adapter * feat(gdrive): normalize internal doc links, tabs, and header/footer structure * fix(gdrive): reject generic source credential flags * test(gdrive): include local adapter in expected list * fix(gdrive): remove dead exports and silence false positive secret checks * fix(setup): restore notion source auth flow
This commit is contained in:
ARYAN
GitHub
parent
967a413a06
commit
5645dc4d28
39 changed files with 2546 additions and 74 deletions
|
|
@ -3,8 +3,16 @@ import { tmpdir } from 'node:os';
|
|||
import { join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { localConnectionTypeForConfig } from './context/connections/local-warehouse-descriptor.js';
|
||||
import {
|
||||
parseGdriveConnectionConfig,
|
||||
resolveGdriveServiceAccountKey,
|
||||
} from './context/connections/gdrive-config.js';
|
||||
import { resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
|
||||
import { resolveKtxConfigReference } from './context/core/config-reference.js';
|
||||
import {
|
||||
createGoogleDocsClients,
|
||||
} from './context/ingest/adapters/gdrive/gdrive-client.js';
|
||||
import { GDRIVE_DOC_MIME_TYPE, gdriveServiceAccountKeySchema } from './context/ingest/adapters/gdrive/types.js';
|
||||
import { cloneOrPull, testRepoConnection } from './context/ingest/repo-fetch.js';
|
||||
import { DEFAULT_METABASE_CLIENT_CONFIG, MetabaseClient } from './context/ingest/adapters/metabase/client.js';
|
||||
import { discoverMetabaseDatabases, type DiscoveredMetabaseDatabase } from './context/ingest/adapters/metabase/mapping.js';
|
||||
|
|
@ -37,7 +45,7 @@ import {
|
|||
type KtxSetupPromptOption,
|
||||
} from './setup-prompts.js';
|
||||
|
||||
export type KtxSetupSourceType = 'dbt' | 'metricflow' | 'metabase' | 'looker' | 'lookml' | 'notion';
|
||||
export type KtxSetupSourceType = 'dbt' | 'metricflow' | 'metabase' | 'looker' | 'lookml' | 'notion' | 'gdrive';
|
||||
|
||||
const DEFAULT_NOTION_MAX_KNOWLEDGE_CREATES_PER_RUN = 25;
|
||||
|
||||
|
|
@ -62,6 +70,9 @@ export interface KtxSetupSourcesArgs {
|
|||
metabaseDatabaseId?: number;
|
||||
notionCrawlMode?: 'all_accessible' | 'selected_roots';
|
||||
notionRootPageIds?: string[];
|
||||
gdriveServiceAccountKeyRef?: string;
|
||||
gdriveFolderId?: string;
|
||||
gdriveRecursive?: boolean;
|
||||
runInitialSourceIngest: boolean;
|
||||
skipSources: boolean;
|
||||
}
|
||||
|
|
@ -103,6 +114,7 @@ export interface KtxSetupSourcesDeps {
|
|||
validateLooker?: (projectDir: string, connectionId: string) => Promise<SourceValidationResult>;
|
||||
validateLookml?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
validateNotion?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
validateGdrive?: (connection: KtxProjectConnectionConfig) => Promise<SourceValidationResult>;
|
||||
pickNotionRootPages?: typeof pickNotionRootPages;
|
||||
discoverMetabaseDatabases?: (args: {
|
||||
sourceUrl: string;
|
||||
|
|
@ -125,6 +137,7 @@ const SOURCE_OPTIONS: Array<{ value: KtxSetupSourceType; label: string }> = [
|
|||
{ value: 'metricflow', label: 'MetricFlow' },
|
||||
{ value: 'looker', label: 'Looker' },
|
||||
{ value: 'lookml', label: 'LookML' },
|
||||
{ value: 'gdrive', label: 'Google Drive' },
|
||||
];
|
||||
|
||||
const SOURCE_LABELS = Object.fromEntries(SOURCE_OPTIONS.map((option) => [option.value, option.label])) as Record<
|
||||
|
|
@ -218,8 +231,10 @@ function credentialRef(value: string | undefined, label: string): string {
|
|||
return ref;
|
||||
}
|
||||
|
||||
type SharedSourceCredentialField = 'sourceAuthTokenRef' | 'sourceApiKeyRef' | 'sourceClientSecretRef';
|
||||
|
||||
type SourceCredentialFlag = {
|
||||
field: 'sourceAuthTokenRef' | 'sourceApiKeyRef' | 'sourceClientSecretRef';
|
||||
field: SharedSourceCredentialField | null;
|
||||
flag: string;
|
||||
};
|
||||
|
||||
|
|
@ -232,9 +247,10 @@ const SOURCE_CREDENTIAL_FLAG: Record<KtxSetupSourceType, SourceCredentialFlag> =
|
|||
notion: { field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
||||
looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
||||
gdrive: { field: null, flag: '--gdrive-service-account-key-ref' },
|
||||
};
|
||||
|
||||
const ALL_SOURCE_CREDENTIAL_FLAGS: SourceCredentialFlag[] = [
|
||||
const ALL_SOURCE_CREDENTIAL_FLAGS: Array<{ field: SharedSourceCredentialField; flag: string }> = [
|
||||
{ field: 'sourceAuthTokenRef', flag: '--source-auth-token-ref' },
|
||||
{ field: 'sourceApiKeyRef', flag: '--source-api-key-ref' },
|
||||
{ field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' },
|
||||
|
|
@ -560,6 +576,22 @@ function buildNotionConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionC
|
|||
};
|
||||
}
|
||||
|
||||
function buildGdriveConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionConfig {
|
||||
const folderId = args.gdriveFolderId?.trim();
|
||||
if (!folderId) {
|
||||
throw new Error('Google Drive setup requires --gdrive-folder-id.');
|
||||
}
|
||||
return {
|
||||
driver: 'gdrive',
|
||||
service_account_key_ref: credentialRef(
|
||||
args.gdriveServiceAccountKeyRef,
|
||||
'Google Drive service account key ref',
|
||||
),
|
||||
folder_id: folderId,
|
||||
recursive: args.gdriveRecursive === true,
|
||||
};
|
||||
}
|
||||
|
||||
function sourcePathFromFileRepoUrl(repoUrl: string, subpath?: string): string {
|
||||
const root = fileURLToPath(repoUrl);
|
||||
return subpath ? join(root, subpath) : root;
|
||||
|
|
@ -680,6 +712,17 @@ async function defaultValidateNotion(connection: KtxProjectConnectionConfig): Pr
|
|||
return { ok: true, detail: `roots=${roots.length}` };
|
||||
}
|
||||
|
||||
async function defaultValidateGdrive(connection: KtxProjectConnectionConfig): Promise<SourceValidationResult> {
|
||||
const config = parseGdriveConnectionConfig(connection);
|
||||
const keyText = await resolveGdriveServiceAccountKey(config.service_account_key_ref);
|
||||
const clients = createGoogleDocsClients(gdriveServiceAccountKeySchema.parse(JSON.parse(keyText)));
|
||||
const result = await clients.drive.listFiles({
|
||||
q: `'${config.folder_id}' in parents and trashed = false`,
|
||||
});
|
||||
const docs = result.files.filter((file) => file.mimeType === GDRIVE_DOC_MIME_TYPE).length;
|
||||
return { ok: true, detail: `docs=${docs}` };
|
||||
}
|
||||
|
||||
interface MappingJsonOutput {
|
||||
connectionId: string;
|
||||
refresh: { ok: boolean; output: string[] };
|
||||
|
|
@ -1329,67 +1372,105 @@ async function promptForInteractiveSource(
|
|||
]);
|
||||
}
|
||||
|
||||
return await runSourcePromptSteps(initialState, (state) => [
|
||||
if (source === 'notion') {
|
||||
return await runSourcePromptSteps(initialState, (state) => [
|
||||
...connectionSteps,
|
||||
async (currentState) => {
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Notion integration token',
|
||||
envName: 'NOTION_TOKEN',
|
||||
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
||||
existingRef: currentState.sourceAuthTokenRef,
|
||||
});
|
||||
if (ref === 'back') return 'back';
|
||||
currentState.sourceAuthTokenRef = ref;
|
||||
return 'next';
|
||||
},
|
||||
async (currentState) => {
|
||||
const crawlMode = await prompts.select({
|
||||
message: 'Which Notion pages should ktx ingest?',
|
||||
options: [
|
||||
{ value: 'all_accessible', label: 'All pages the integration can access' },
|
||||
{ value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (crawlMode === 'back') return 'back';
|
||||
currentState.notionCrawlMode = crawlMode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
||||
if (currentState.notionCrawlMode === 'all_accessible') {
|
||||
delete currentState.notionRootPageIds;
|
||||
}
|
||||
return 'next';
|
||||
},
|
||||
...(state.notionCrawlMode === 'selected_roots'
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
const connectionId = currentState.sourceConnectionId ?? 'notion-main';
|
||||
const result = await (deps.pickNotionRootPages ?? pickNotionRootPages)(
|
||||
{
|
||||
connectionId,
|
||||
connection: {
|
||||
driver: 'notion',
|
||||
auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'),
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: currentState.notionRootPageIds ?? [],
|
||||
root_database_ids: [],
|
||||
root_data_source_ids: [],
|
||||
},
|
||||
},
|
||||
io,
|
||||
);
|
||||
if (result.kind === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
if (result.kind === 'unavailable') {
|
||||
io.stderr.write(`${result.message}\n`);
|
||||
return 'back';
|
||||
}
|
||||
currentState.notionRootPageIds = result.rootPageIds;
|
||||
return 'next';
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
}
|
||||
|
||||
return await runSourcePromptSteps(initialState, () => [
|
||||
...connectionSteps,
|
||||
async (currentState) => {
|
||||
const ref = await chooseSourceCredentialRef({
|
||||
prompts,
|
||||
projectDir: args.projectDir,
|
||||
label: 'Notion integration token',
|
||||
envName: 'NOTION_TOKEN',
|
||||
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
|
||||
existingRef: currentState.sourceAuthTokenRef,
|
||||
const keyRef = await promptText(prompts, {
|
||||
message: 'Google Drive service account key file reference',
|
||||
placeholder: 'file:/absolute/path/to/key.json',
|
||||
...(currentState.gdriveServiceAccountKeyRef ? { initialValue: currentState.gdriveServiceAccountKeyRef } : {}),
|
||||
});
|
||||
if (ref === 'back') return 'back';
|
||||
currentState.sourceAuthTokenRef = ref;
|
||||
if (keyRef === undefined) return 'back';
|
||||
currentState.gdriveServiceAccountKeyRef = keyRef.trim();
|
||||
return 'next';
|
||||
},
|
||||
async (currentState) => {
|
||||
const crawlMode = await prompts.select({
|
||||
message: 'Which Notion pages should ktx ingest?',
|
||||
const folderId = await promptText(prompts, {
|
||||
message: 'Google Drive folder id',
|
||||
...(currentState.gdriveFolderId ? { initialValue: currentState.gdriveFolderId } : {}),
|
||||
});
|
||||
if (folderId === undefined) return 'back';
|
||||
currentState.gdriveFolderId = folderId.trim();
|
||||
return 'next';
|
||||
},
|
||||
async (currentState) => {
|
||||
const recursive = await prompts.select({
|
||||
message: 'Include Google Docs from subfolders?',
|
||||
options: [
|
||||
{ value: 'all_accessible', label: 'All pages the integration can access' },
|
||||
{ value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' },
|
||||
{ value: 'false', label: 'No' },
|
||||
{ value: 'true', label: 'Yes' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
if (crawlMode === 'back') return 'back';
|
||||
currentState.notionCrawlMode = crawlMode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
||||
if (currentState.notionCrawlMode === 'all_accessible') {
|
||||
delete currentState.notionRootPageIds;
|
||||
}
|
||||
if (recursive === 'back') return 'back';
|
||||
currentState.gdriveRecursive = recursive === 'true';
|
||||
return 'next';
|
||||
},
|
||||
...(state.notionCrawlMode === 'selected_roots'
|
||||
? [
|
||||
async (currentState: SourcePromptState) => {
|
||||
const connectionId = currentState.sourceConnectionId ?? 'notion-main';
|
||||
const result = await (deps.pickNotionRootPages ?? pickNotionRootPages)(
|
||||
{
|
||||
connectionId,
|
||||
connection: {
|
||||
driver: 'notion',
|
||||
auth_token_ref: credentialRef(currentState.sourceAuthTokenRef, 'Notion token ref'),
|
||||
crawl_mode: 'selected_roots',
|
||||
root_page_ids: currentState.notionRootPageIds ?? [],
|
||||
root_database_ids: [],
|
||||
root_data_source_ids: [],
|
||||
},
|
||||
},
|
||||
io,
|
||||
);
|
||||
if (result.kind === 'back') {
|
||||
return 'back';
|
||||
}
|
||||
if (result.kind === 'unavailable') {
|
||||
io.stderr.write(`${result.message}\n`);
|
||||
return 'back';
|
||||
}
|
||||
currentState.notionRootPageIds = result.rootPageIds;
|
||||
return 'next';
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
@ -1559,6 +1640,13 @@ function sourceArgsFromExistingConnection(input: {
|
|||
return sourceArgs;
|
||||
}
|
||||
|
||||
if (input.source === 'gdrive') {
|
||||
sourceArgs.gdriveServiceAccountKeyRef = stringField(input.connection.service_account_key_ref);
|
||||
sourceArgs.gdriveFolderId = stringField(input.connection.folder_id);
|
||||
sourceArgs.gdriveRecursive = input.connection.recursive === true;
|
||||
return sourceArgs;
|
||||
}
|
||||
|
||||
sourceArgs.sourceAuthTokenRef = stringField(input.connection.auth_token_ref);
|
||||
sourceArgs.notionCrawlMode =
|
||||
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
|
||||
|
|
@ -1740,7 +1828,10 @@ function buildConnection(source: KtxSetupSourceType, args: KtxSetupSourcesArgs):
|
|||
if (source === 'lookml') {
|
||||
return buildLookmlConnection(args);
|
||||
}
|
||||
return buildNotionConnection(args);
|
||||
if (source === 'notion') {
|
||||
return buildNotionConnection(args);
|
||||
}
|
||||
return buildGdriveConnection(args);
|
||||
}
|
||||
|
||||
async function validateSource(
|
||||
|
|
@ -1765,7 +1856,10 @@ async function validateSource(
|
|||
if (source === 'lookml') {
|
||||
return await (deps.validateLookml ?? defaultValidateLookml)(args.connection);
|
||||
}
|
||||
return await (deps.validateNotion ?? defaultValidateNotion)(args.connection);
|
||||
if (source === 'notion') {
|
||||
return await (deps.validateNotion ?? defaultValidateNotion)(args.connection);
|
||||
}
|
||||
return await (deps.validateGdrive ?? defaultValidateGdrive)(args.connection);
|
||||
}
|
||||
|
||||
async function createSourceSetupRollback(projectDir: string): Promise<() => Promise<void>> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue