feat: Add remote source support for IR migration

.github/workflows/pages.yml

@@ -0,0 +1,51 @@
+# GitHub Pages deployment workflow
+name: Deploy Documentation
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - 'docs/**'
+      - '.github/workflows/pages.yml'
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./docs
+          destination: ./_site
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

crates/morphir-common/Cargo.toml

@@ -19,6 +19,14 @@ toml = "0.8"
 dirs = "6.0"
 morphir-ir = { path = "../morphir-ir" }
 glob = "0.3"
+indexmap = { version = "2", features = ["serde"] }
+
+# Remote source support
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+flate2 = "1.0"
+zip = "2.1"
+tar = "0.4"
+tempfile = "3.24"
 
 [dev-dependencies]
 tempfile = "3.24.0"

crates/morphir-common/src/config/model.rs

@@ -1,3 +1,4 @@
+use crate::remote::config::RemoteSourceConfig;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::PathBuf;
@@ -29,6 +30,10 @@ pub struct MorphirConfig {
     #[serde(default)]
     pub codegen: Option<CodegenSection>,
 
+    /// Remote source configuration
+    #[serde(default)]
+    pub sources: Option<RemoteSourceConfig>,
+
     /// Dependencies
     #[serde(default)]
     pub dependencies: HashMap<String, DependencySpec>,

crates/morphir-common/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod config;
 pub mod loader;
 pub mod pipeline;
+pub mod remote;
 pub mod vfs;
 
 pub type Result<T> = anyhow::Result<T>;

crates/morphir-common/src/loader.rs

@@ -1,24 +1,81 @@
-use crate::vfs::Vfs;
+use crate::remote::{RemoteSource, RemoteSourceResolver, ResolveOptions};
+use crate::vfs::{OsVfs, Vfs};
 use anyhow::{Context, Result};
+use indexmap::IndexMap;
 use morphir_ir::ir::{classic, v4};
+use morphir_ir::naming::PackageName;
 use std::path::Path;
 
 #[derive(Debug)]
 pub enum LoadedDistribution {
-    V4(v4::Distribution),
+    V4(v4::IRFile),
     Classic(classic::Distribution),
 }
 
+/// Load distribution from a source string (local path or remote source).
+///
+/// This is a convenience function that:
+/// 1. Parses the source string into a RemoteSource
+/// 2. Resolves it to a local path (downloading if necessary)
+/// 3. Loads the distribution from the local path
+///
+/// # Arguments
+/// * `source` - A source string (local path, URL, or shorthand like `github:owner/repo`)
+///
+/// # Examples
+/// ```ignore
+/// // Local file
+/// let dist = load_distribution_from_source("./morphir-ir.json")?;
+///
+/// // Remote URL
+/// let dist = load_distribution_from_source("https://example.com/morphir-ir.json")?;
+///
+/// // GitHub shorthand
+/// let dist = load_distribution_from_source("github:finos/morphir-examples/examples/basic")?;
+/// ```
+pub fn load_distribution_from_source(source: &str) -> Result<LoadedDistribution> {
+    load_distribution_from_source_with_options(source, &ResolveOptions::new())
+}
+
+/// Load distribution from a source string with custom resolve options.
+pub fn load_distribution_from_source_with_options(
+    source: &str,
+    options: &ResolveOptions,
+) -> Result<LoadedDistribution> {
+    let remote_source =
+        RemoteSource::parse(source).map_err(|e| anyhow::anyhow!("Invalid source: {}", e))?;
+
+    let local_path = if remote_source.is_local() {
+        std::path::PathBuf::from(source)
+    } else {
+        let mut resolver = RemoteSourceResolver::with_defaults()
+            .map_err(|e| anyhow::anyhow!("Failed to create source resolver: {}", e))?;
+
+        resolver
+            .resolve(&remote_source, options)
+            .map_err(|e| anyhow::anyhow!("Failed to resolve source: {}", e))?
+    };
+
+    let vfs = OsVfs;
+    load_distribution(&vfs, &local_path)
+}
+
 pub fn load_distribution(vfs: &impl Vfs, path: &Path) -> Result<LoadedDistribution> {
     if vfs.is_dir(path) {
         return load_v4_from_dir(vfs, path);
     }
 
     let content = vfs.read_to_string(path)?;
 
-    if let Ok(dist) = serde_json::from_str::<v4::Distribution>(&content) {
-        if dist.format_version == 4 {
-            return Ok(LoadedDistribution::V4(dist));
+    if let Ok(ir_file) = serde_json::from_str::<v4::IRFile>(&content) {
+        // Check if it's a V4 format based on format_version
+        let is_v4 = match &ir_file.format_version {
+            v4::FormatVersion::Integer(n) => *n >= 4,
+            v4::FormatVersion::String(s) => s.starts_with("4"),
+        };
+
+        if is_v4 {
+            return Ok(LoadedDistribution::V4(ir_file));
         }
     }
 
@@ -46,7 +103,7 @@ fn load_v4_from_dir(vfs: &impl Vfs, path: &Path) -> Result<LoadedDistribution> {
 
     // Scan for module JSON files in src/ directory
     let src_path = path.join("src");
-    let mut modules = Vec::new();
+    let mut modules: IndexMap<String, v4::AccessControlledModuleDefinition> = IndexMap::new();
 
     if vfs.is_dir(&src_path) {
         // Use glob to find all JSON files under src/
@@ -70,30 +127,26 @@ fn load_v4_from_dir(vfs: &impl Vfs, path: &Path) -> Result<LoadedDistribution> {
                 .to_string_lossy()
                 .replace('/', ".");
 
-            let module_entry = v4::ModuleDefinitionEntry(
-                v4::Path::new(&module_name),
-                v4::AccessControlledModuleDefinition {
-                    access: v4::Access::Public,
-                    value: v4::ModuleDefinition {
-                        types: vec![],
-                        values: vec![],
-                        doc: None,
-                    },
+            let module_def = v4::AccessControlledModuleDefinition {
+                access: v4::Access::Public,
+                value: v4::ModuleDefinition {
+                    types: IndexMap::new(),
+                    values: IndexMap::new(),
+                    doc: None,
                 },
-            );
-            modules.push(module_entry);
+            };
+            modules.insert(module_name, module_def);
         }
     }
 
-    let dist = v4::Distribution {
-        format_version: 4,
-        distribution: v4::DistributionBody::Library(v4::LibraryDistribution(
-            v4::LibraryTag::Library,
-            v4::Path::new(&package_name),
-            vec![],
-            v4::PackageDefinition { modules },
-        )),
+    let ir_file = v4::IRFile {
+        format_version: v4::FormatVersion::default(),
+        distribution: v4::Distribution::Library(v4::LibraryContent {
+            package_name: PackageName::parse(&package_name),
+            dependencies: IndexMap::new(),
+            def: v4::PackageDefinition { modules },
+        }),
     };
 
-    Ok(LoadedDistribution::V4(dist))
+    Ok(LoadedDistribution::V4(ir_file))
 }