Add file loader

packages/astro/src/content/file.ts

@@ -1,10 +1,46 @@
-/// <reference types="../../types/content.d.ts" />
-import type { Loader } from 'astro:content';
+import type { Loader } from './loaders.js';
+import { promises as fs, existsSync } from 'fs';
 
 export function file(fileName: string | URL): Loader {
 	return {
-		name: 'post-loader',
-		load: async ({ store, logger }) => {
+		name: 'file-loader',
+		load: async ({ store, logger, settings }) => {
+			const contentDir = new URL('./content/', settings.config.srcDir);
+
+			const url = new URL(fileName, contentDir);
+			if (!existsSync(url)) {
+				logger.error(`File not found: ${fileName}`);
+				return;
+			}
+
+			const data = await fs.readFile(url, 'utf-8');
+			const json = JSON.parse(data);
+
+			if (Array.isArray(json)) {
+				if (json.length === 0) {
+					logger.warn(`No items found in ${fileName}`);
+				} else {
+					const firstItem = json[0];
+					if (!firstItem.id && !firstItem.slug) {
+						logger.error(`Invalid data in ${fileName}. Array items have an id or slug field.`);
+					}
+				}
+				for (const item of json) {
+					const id = item.id ?? item.slug;
+					if (!id) {
+						logger.debug(`Missing ID or slug for item: ${JSON.stringify(item)}`);
+						continue;
+					}
+					store.set(id, item);
+				}
+			} else if (typeof json === 'object') {
+				for (const [id, item] of Object.entries(json)) {
+					store.set(id, item);
+				}
+			} else {
+				logger.error(`Invalid data in ${fileName}. Must be an array or object.`);
+			}
+
 			logger.info('Loading posts');
 		},
 	};

packages/astro/src/content/loaders.ts

@@ -1,8 +1,31 @@
+import type { ZodSchema } from 'zod';
 import type { AstroSettings } from '../@types/astro.js';
-import type { Logger } from '../core/logger/core.js';
-import { DataStore, globalDataStore } from './data-store.js';
+import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js';
+import { DataStore, globalDataStore, type MetaStore, type ScopedDataStore } from './data-store.js';
 import { globalContentConfigObserver } from './utils.js';
+import { promises as fs, existsSync } from 'fs';
 
+export interface LoaderContext {
+	collection: string;
+	// A database abstraction to store the actual data
+	store: ScopedDataStore;
+	// A simple KV store, designed for things like sync tokens
+	meta: MetaStore;
+	logger: AstroIntegrationLogger;
+
+	settings: AstroSettings;
+}
+
+export interface Loader<S extends ZodSchema = ZodSchema> {
+	// Name of the loader, e.g. the npm package name
+	name: string;
+	// Do the actual loading of the data
+	load: (context: LoaderContext) => Promise<void>;
+	// Allow a loader to define its own schema
+	schema?: S | Promise<S> | (() => S | Promise<S>);
+	// Render content from the store
+	render?: (entry: any) => any;
+}
 export async function syncDataLayer({
 	settings,
 	logger,
@@ -27,10 +50,14 @@ export async function syncDataLayer({
 				store: store.scopedStore(name),
 				meta: store.metaStore(name),
 				logger: logger.forkIntegrationLogger('content'),
+				settings,
 			});
 		})
 	);
 	const cacheFile = new URL('data-store.json', settings.config.cacheDir);
+	if (!existsSync(settings.config.cacheDir)) {
+		await fs.mkdir(settings.config.cacheDir, { recursive: true });
+	}
 	await store.writeToDisk(cacheFile);
 	logger.info(null, 'Synced data layer');
 }

packages/astro/src/content/runtime.ts

@@ -15,6 +15,7 @@ import {
 } from '../runtime/server/index.js';
 import type { ContentLookupMap } from './utils.js';
 import { globalDataStore } from './data-store.js';
+export { file } from './file.js';
 
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;

packages/astro/src/content/types-generator.ts

@@ -435,7 +435,11 @@ async function writeContentFiles({
 				: collection.type;
 
 		const collectionEntryKeys = Object.keys(collection.entries).sort();
-		const dataType = collectionConfig?.schema ? `InferEntrySchema<${collectionKey}>` : 'any';
+		const dataType =
+			collectionConfig?.schema ||
+			(collectionConfig?.type === 'experimental_data' && collectionConfig.loader?.schema)
+				? `InferEntrySchema<${collectionKey}>`
+				: 'any';
 		switch (resolvedType) {
 			case 'content':
 				if (collectionEntryKeys.length === 0) {
@@ -455,6 +459,7 @@ async function writeContentFiles({
 				contentTypesStr += `};\n`;
 				break;
 			case 'data':
+			case 'experimental_data':
 				if (collectionEntryKeys.length === 0) {
 					dataTypesStr += `${collectionKey}: Record<string, {\n  id: string;\n  collection: ${collectionKey};\n  data: ${dataType};\n}>;\n`;
 				} else {
@@ -496,9 +501,6 @@ async function writeContentFiles({
 					}
 				}
 				break;
-			case 'experimental_data':
-				console.log('experimental_data', collectionKey);
-				break;
 		}
 	}
 

packages/astro/src/content/utils.ts

@@ -37,8 +37,7 @@ export const collectionConfigParser = z.union([
 	}),
 	z.object({
 		type: z.literal('experimental_data'),
-		name: z.string(),
-		schema: z.undefined().optional(),
+		schema: z.any().optional(),
 		loader: z.object({
 			name: z.string(),
 			load: z.function(
@@ -49,6 +48,7 @@ export const collectionConfigParser = z.union([
 							store: z.any(),
 							meta: z.any(),
 							logger: z.any(),
+							settings: z.any(),
 						}),
 					],
 					z.unknown()