Skip to content

Commit

Permalink
update docs generation workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
@kMutagene
bvenn authored and kMutagene committed Feb 5, 2021
1 parent ff4ff53 commit 36c34bf
Show file tree
Hide file tree
Showing 8 changed files with 9,223 additions and 256 deletions.
6 changes: 6 additions & 0 deletions .config/dotnet-tools.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
"commands": [
"fake"
]
},
"fsharp.formatting.commandtool": {
"version": "8.0.0",
"commands": [
"fsdocs"
]
}
}
}
6 changes: 5 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,12 @@ docsrc/content/release-notes.md
docsrc/tools/FSharp.Formatting.svclog

# generated docs
docs/
/temp
/output
.fsdocs
/tmp/watch

/codeCov.xml
/tests/FSharp.Stats.Tests/coverage.xml

.ionide
38 changes: 0 additions & 38 deletions FSharp.Stats.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29806.167
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docsrc", "docsrc", "{A6A6AF7D-D6E3-442D-9B1E-58CC91879BE1}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Stats", "src\FSharp.Stats\FSharp.Stats.fsproj", "{FAAFA66E-170B-4DA5-AA21-69A429CBE646}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "project", "project", "{BF60BC93-E09B-4E5F-9D85-95A519479D54}"
Expand All @@ -12,40 +10,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "project", "project", "{BF60
RELEASE_NOTES.md = RELEASE_NOTES.md
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{83F16175-43B1-4C90-A1EE-8E351C33435D}"
ProjectSection(SolutionItems) = preProject
docsrc\tools\templates\template.cshtml = docsrc\tools\templates\template.cshtml
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "content", "content", "{8E6D5255-776D-4B61-85F9-73C37AA1FB9A}"
ProjectSection(SolutionItems) = preProject
docsrc\content\BasicStats.fsx = docsrc\content\BasicStats.fsx
docsrc\content\Clustering.fsx = docsrc\content\Clustering.fsx
docsrc\content\Correlation.fsx = docsrc\content\Correlation.fsx
docsrc\content\Covariance.fsx = docsrc\content\Covariance.fsx
docsrc\content\CrossValidation.fsx = docsrc\content\CrossValidation.fsx
docsrc\content\Distributions.fsx = docsrc\content\Distributions.fsx
docsrc\content\Fitting.fsx = docsrc\content\Fitting.fsx
docsrc\content\GoodnessOfFit.fsx = docsrc\content\GoodnessOfFit.fsx
docsrc\content\GrowthCurve.fsx = docsrc\content\GrowthCurve.fsx
docsrc\content\Imputation.fsx = docsrc\content\Imputation.fsx
docsrc\content\index.fsx = docsrc\content\index.fsx
docsrc\content\Integration.fsx = docsrc\content\Integration.fsx
docsrc\content\Interpolation.fsx = docsrc\content\Interpolation.fsx
docsrc\content\license.md = docsrc\content\license.md
docsrc\content\LinearAlgebra.fsx = docsrc\content\LinearAlgebra.fsx
docsrc\content\Matrix_Vector.fsx = docsrc\content\Matrix_Vector.fsx
docsrc\content\ML.fsx = docsrc\content\ML.fsx
docsrc\content\Quantile.fsx = docsrc\content\Quantile.fsx
docsrc\content\Rank.fsx = docsrc\content\Rank.fsx
docsrc\content\release-notes.md = docsrc\content\release-notes.md
docsrc\content\Signal.fsx = docsrc\content\Signal.fsx
docsrc\content\SpecialFunctions.fsx = docsrc\content\SpecialFunctions.fsx
docsrc\content\Summary.fsx = docsrc\content\Summary.fsx
docsrc\content\Testing.fsx = docsrc\content\Testing.fsx
docsrc\content\tutorial.fsx = docsrc\content\tutorial.fsx
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{ED8079DD-2B06-4030-9F0F-DC548F98E1C4}"
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Stats.Tests", "tests\FSharp.Stats.Tests\FSharp.Stats.Tests.fsproj", "{DA7E2263-283A-497F-B122-B2E1A5AC2C8C}"
Expand Down Expand Up @@ -87,8 +51,6 @@ Global
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{83F16175-43B1-4C90-A1EE-8E351C33435D} = {A6A6AF7D-D6E3-442D-9B1E-58CC91879BE1}
{8E6D5255-776D-4B61-85F9-73C37AA1FB9A} = {A6A6AF7D-D6E3-442D-9B1E-58CC91879BE1}
{DA7E2263-283A-497F-B122-B2E1A5AC2C8C} = {ED8079DD-2B06-4030-9F0F-DC548F98E1C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
Expand Down
236 changes: 21 additions & 215 deletions build.fsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,60 +34,10 @@ Target.initEnvironment ()

let release = ReleaseNotes.load "RELEASE_NOTES.md"

[<AutoOpen>]
module TemporaryDocumentationHelpers =

type LiterateArguments =
{ ToolPath : string
Source : string
OutputDirectory : string
Template : string
ProjectParameters : (string * string) list
LayoutRoots : string list
FsiEval : bool }


let private run toolPath command =
if 0 <> Process.execSimple ((fun info ->
{ info with
FileName = toolPath
Arguments = command }) >> Process.withFramework) System.TimeSpan.MaxValue

then failwithf "FSharp.Formatting %s failed." command

let createDocs p =
let toolPath = Tools.findToolInSubPath "fsformatting.exe" (Directory.GetCurrentDirectory() @@ "lib/Formatting")
let defaultLiterateArguments =
{ ToolPath = toolPath
Source = ""
OutputDirectory = ""
Template = ""
ProjectParameters = []
LayoutRoots = []
FsiEval = false }

let arguments = (p:LiterateArguments->LiterateArguments) defaultLiterateArguments
let layoutroots =
if arguments.LayoutRoots.IsEmpty then []
else [ "--layoutRoots" ] @ arguments.LayoutRoots
let source = arguments.Source
let template = arguments.Template
let outputDir = arguments.OutputDirectory
let fsiEval = if arguments.FsiEval then [ "--fsieval" ] else []

let command =
arguments.ProjectParameters
|> Seq.map (fun (k, v) -> [ k; v ])
|> Seq.concat
|> Seq.append
(["literate"; "--processdirectory" ] @ layoutroots @ [ "--inputdirectory"; source; "--templatefile"; template;
"--outputDirectory"; outputDir] @ fsiEval @ [ "--replacements" ])
|> Seq.map (fun s ->
if s.StartsWith "\"" then s
else sprintf "\"%s\"" s)
|> String.separated " "
run arguments.ToolPath command
printfn "Successfully generated docs for %s" source
let runDotNet cmd workingDir =
let result =
DotNet.exec (DotNet.Options.withWorkingDirectory workingDir) cmd ""
if result.ExitCode <> 0 then failwithf "'dotnet %s' failed in %s" cmd workingDir

[<AutoOpen>]
module MessagePrompts =
Expand Down Expand Up @@ -123,7 +73,7 @@ let owners = "fslaborg, Timo Muehlhaus"
let description = "F#-first linear algebra, machine learning, fitting, signal processing, and statistical testing."
let licenseUrl = "https://github.com/fslaborg/FSharp.Stats/blob/developer/LICENSE"
let projectUrl = "https://github.com/fslaborg/FSharp.Stats"
let iconUrl = "http://raw.githubusercontent.com/fslaborg/FSharp.Stats/developer/docsrc/files/img/logo.png"
let iconUrl = "http://raw.githubusercontent.com/fslaborg/FSharp.Stats/developer/docs/img/logo.png"
let tags = "F# FSharp dotnet data-science linear-algebra machine-learning fitting signal-processing statistical-testing"
let releaseNotes = (release.Notes |> String.concat "\r\n")
let repositoryUrl = "https://github.com/fslaborg/FSharp.Stats"
Expand All @@ -139,13 +89,6 @@ let clean = BuildTask.create "Clean" [] {
|> Shell.cleanDirs
}

let cleanDocs = BuildTask.create "CleanDocs" [] {
!! "docs"
++ "temp"
|> Shell.cleanDirs
}


let build = BuildTask.create "Build" [clean] {
!! "src/**/*.*proj"
|> Seq.iter (DotNet.build id)
Expand All @@ -169,17 +112,7 @@ let pack = BuildTask.create "Pack" [clean; build] {
{p.MSBuildParams with
Properties = ([
"Version",(sprintf "%i.%i.%i" stableVersion.Major stableVersion.Minor stableVersion.Patch )
"Authors", authors
"Title", title
"Owners", owners
"Description", description
"PackageLicenseUrl", licenseUrl
"PackageProjectUrl", projectUrl
"PackageIconUrl", iconUrl
"PackageTags", tags
"PackageReleaseNotes", releaseNotes
"RepositoryUrl", repositoryUrl
"RepositoryType", "git"
] @ p.MSBuildParams.Properties)
}
{
Expand All @@ -203,17 +136,7 @@ let packPrerelease = BuildTask.create "PackPrerelease" [clean; build] {
{p.MSBuildParams with
Properties = ([
"Version", prereleaseTag
"Authors", authors
"Title", title
"Owners", owners
"Description", description
"PackageLicenseUrl", licenseUrl
"PackageProjectUrl", projectUrl
"PackageIconUrl", iconUrl
"PackageTags", tags
"PackageReleaseNotes", releaseNotes
"RepositoryUrl", repositoryUrl
"RepositoryType", "git"
] @ p.MSBuildParams.Properties)
}
{
Expand All @@ -227,145 +150,28 @@ let packPrerelease = BuildTask.create "PackPrerelease" [clean; build] {
))
}



// --------------------------------------------------------------------------------------
// Generate the documentation

// Paths with template/source/output locations
let bin = __SOURCE_DIRECTORY__ @@ "bin"
let content = __SOURCE_DIRECTORY__ @@ "docsrc/content"
let output = __SOURCE_DIRECTORY__ @@ "docs"
let files = __SOURCE_DIRECTORY__ @@ "docsrc/files"
let templates = __SOURCE_DIRECTORY__ @@ "docsrc/tools/templates"
let formatting = __SOURCE_DIRECTORY__ @@ "packages/formatting/FSharp.Formatting"
let docTemplate = "docpage.cshtml"

let github_release_user = Environment.environVarOrDefault "github_release_user" gitOwner
let githubLink = sprintf "https://github.com/%s/%s" github_release_user gitName

// Specify more information about your project
let info =
[ "project-name", title
"project-author", authors
"project-summary", description
"project-github", githubLink
"project-nuget", "http://nuget.org/packages/FSharp.Stats" ]

let root = website

let referenceBinaries = []

let layoutRootsAll = new System.Collections.Generic.Dictionary<string, string list>()
layoutRootsAll.Add("en",[ templates;
formatting @@ "templates"
formatting @@ "templates/reference" ])


let docs = BuildTask.create "Docs" [cleanDocs; build; copyBinaries] {
let copyFiles () =
Shell.copyRecursive files output true
|> Trace.logItems "Copying file: "
Directory.ensure (output @@ "content")
Shell.copyRecursive (formatting @@ "styles") (output @@ "content") true
|> Trace.logItems "Copying styles and scripts: "

File.delete "docsrc/content/release-notes.md"
Shell.copyFile "docsrc/content/" "RELEASE_NOTES.md"
Shell.rename "docsrc/content/release-notes.md" "docsrc/content/RELEASE_NOTES.md"

File.delete "docsrc/content/license.md"
Shell.copyFile "docsrc/content/" "LICENSE"
Shell.rename "docsrc/content/license.md" "docsrc/content/LICENSE"

DirectoryInfo.getSubDirectories (DirectoryInfo.ofPath templates)
|> Seq.iter (fun d ->
let name = d.Name
if name.Length = 2 || name.Length = 3 then
layoutRootsAll.Add(
name, [templates @@ name
formatting @@ "templates"
formatting @@ "templates/reference" ]))
copyFiles ()

for dir in [ content; ] do
let langSpecificPath(lang, path:string) =
path.Split([|'/'; '\\'|], System.StringSplitOptions.RemoveEmptyEntries)
|> Array.exists(fun i -> i = lang)
let layoutRoots =
let key = layoutRootsAll.Keys |> Seq.tryFind (fun i -> langSpecificPath(i, dir))
match key with
| Some lang -> layoutRootsAll.[lang]
| None -> layoutRootsAll.["en"] // "en" is the default language

createDocs (fun args ->
{ args with
Source = content
OutputDirectory = output
LayoutRoots = layoutRoots
ProjectParameters = ("root", root)::info
Template = docTemplate
FsiEval = true
} )
}

let referenceDocs = BuildTask.create "ReferenceDocs" [docs] {
Directory.ensure (output @@ "reference")

let binaries () =
let manuallyAdded =
referenceBinaries
|> List.map (fun b -> bin @@ b)
let conventionBased =
DirectoryInfo.getSubDirectories <| DirectoryInfo bin
|> Array.collect (fun d ->
let name, dInfo =
d.Name,(DirectoryInfo.getSubDirectories d).[0]

dInfo.GetFiles()
|> Array.filter (fun x ->
x.Name.ToLower() = (sprintf "%s.dll" name).ToLower())
|> Array.map (fun x -> x.FullName)
)
|> List.ofArray

conventionBased @ manuallyAdded

binaries()
|> FSFormatting.createDocsForDlls (fun args ->
{ args with
OutputDirectory = output @@ "reference"
LayoutRoots = layoutRootsAll.["en"]
ProjectParameters = ("root", root)::info
SourceRepository = githubLink @@ "tree/master" }
)
// generate the docs
let buildDocs = BuildTask.create "BuildDocs" [build; copyBinaries] {
runDotNet "fsdocs build --eval --property Configuration=Release" "./"
}

let releaseDocsConfirmation = BuildTask.create "ReleaseDocsConfirmation" [] { match promptYesNo releaseDocsMsg with | true -> () |_ -> failwith "Release canceled"}

let releaseDocs = BuildTask.create "ReleaseDocs" [releaseDocsConfirmation; docs] {
let tempDocsDir = "temp/gh-pages"
Shell.cleanDir tempDocsDir
Git.Repository.cloneSingleBranch "" (gitHome + "/" + gitName + ".git") "gh-pages" tempDocsDir

Shell.copyRecursive "docs" tempDocsDir true |> Fake.Core.Trace.tracef "%A"
Git.Staging.stageAll tempDocsDir
Git.Commit.exec tempDocsDir (sprintf "Update generated documentation for version %s" release.NugetVersion)
Git.Branches.push tempDocsDir
let watchDocs = BuildTask.create "WatchDocs" [build; copyBinaries] {
runDotNet "fsdocs watch --eval --property Configuration=Release" "./"
}

let releaseLocal = BuildTask.create "ReleaseLocal" [docs] {
let tempDocsDir = "temp/localDocs"
Shell.cleanDir tempDocsDir |> ignore
Shell.copyRecursive "docs" tempDocsDir true |> printfn "%A"
Shell.replaceInFiles
(seq {
yield "href=\"/" + title + "/","href=\""
yield "src=\"/" + title + "/","src=\""})
(Directory.EnumerateFiles tempDocsDir |> Seq.filter (fun x -> x.EndsWith(".html")))
let releaseDocs = BuildTask.create "ReleaseDocs" [buildDocs] {
Shell.cleanDir "temp"
Git.CommandHelper.runSimpleGitCommand "." (sprintf "clone %s temp/gh-pages --depth 1 -b gh-pages" repositoryUrl) |> ignore
Shell.copyRecursive "output" "temp/gh-pages" true |> printfn "%A"
Git.CommandHelper.runSimpleGitCommand "temp/gh-pages" "add ." |> printfn "%s"
let cmd = sprintf """commit -a -m "Update generated documentation for version %s""" release.NugetVersion
Git.CommandHelper.runSimpleGitCommand "temp/gh-pages" cmd |> printfn "%s"
Git.Branches.push "temp/gh-pages"
}


// --------------------------------------------------------------------------------------
// run test
let runTests = BuildTask.create "RunTests" [clean; build; copyBinaries] {
let standardParams = Fake.DotNet.MSBuild.CliArguments.Create ()
Fake.DotNet.DotNet.test(fun testParams ->
Expand Down Expand Up @@ -393,6 +199,6 @@ let runTestsWithCodeCov = BuildTask.create "RunTestsWithCodeCov" [clean; build;
}
) testProject
}
let _all = BuildTask.createEmpty "All" [clean; cleanDocs; build; copyBinaries; runTestsWithCodeCov; pack; docs; referenceDocs]
let _all = BuildTask.createEmpty "All" [clean; build; copyBinaries; runTestsWithCodeCov; pack; buildDocs]

BuildTask.runOrDefault runTests
Loading

0 comments on commit 36c34bf

Please sign in to comment.