diff --git a/.github/workflows/automatic-doc-checks.yml b/.github/workflows/automatic-doc-checks.yml new file mode 100644 index 00000000..cba712e5 --- /dev/null +++ b/.github/workflows/automatic-doc-checks.yml @@ -0,0 +1,16 @@ +name: Main Documentation Checks + +on: + - push + - pull_request + - workflow_dispatch + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + documentation-checks: + uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main + with: + working-directory: 'docs' diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..c8f8ee69 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,8 @@ +/*env*/ +.sphinx/venv +.sphinx/warnings.txt +.sphinx/.wordlist.dic +_build +.DS_Store +__pycache__ +.idea/ diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml new file mode 100644 index 00000000..45fe3a88 --- /dev/null +++ b/docs/.readthedocs.yaml @@ -0,0 +1,27 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: dirhtml + configuration: docs/conf.py + fail_on_warning: true + +# If using Sphinx, optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/.sphinx/requirements.txt diff --git a/docs/.sphinx/_static/custom.css b/docs/.sphinx/_static/custom.css new file mode 100644 index 00000000..cad94b74 --- /dev/null +++ b/docs/.sphinx/_static/custom.css @@ -0,0 +1,189 @@ +/** Fix the font weight (300 for normal, 400 for slightly bold) **/ + +div.page, h1, h2, h3, h4, h5, h6, .sidebar-tree .current-page>.reference, button, input, optgroup, select, textarea, th.head { + font-weight: 300 +} + +.toc-tree li.scroll-current>.reference, dl.glossary dt, dl.simple dt, dl:not([class]) dt { + font-weight: 400; +} + +/** Table styling **/ + +th.head { + text-transform: uppercase; + font-size: var(--font-size--small); +} + +table.docutils { + border: 0; + box-shadow: none; + width:100%; +} + +table.docutils td, table.docutils th, table.docutils td:last-child, table.docutils th:last-child, table.docutils td:first-child, table.docutils th:first-child { + border-right: none; + border-left: none; +} + +/* Allow to centre text horizontally in table data cells */ +table.align-center { + text-align: center !important; +} + +/** No rounded corners **/ + +.admonition, code.literal, .sphinx-tabs-tab, .sphinx-tabs-panel, .highlight { + border-radius: 0; +} + +/** Admonition styling **/ + +.admonition { + border-top: 1px solid #d9d9d9; + border-right: 1px solid #d9d9d9; + border-bottom: 1px solid #d9d9d9; +} + +/** Color for the "copy link" symbol next to headings **/ + +a.headerlink { + color: var(--color-brand-primary); +} + +/** Line to the left of the current navigation entry **/ + +.sidebar-tree li.current-page { + border-left: 2px solid var(--color-brand-primary); +} + +/** Some tweaks for issue #16 **/ + +[role="tablist"] { + border-bottom: 1px solid var(--color-sidebar-item-background--hover); +} + +.sphinx-tabs-tab[aria-selected="true"] { + border: 0; + border-bottom: 2px solid var(--color-brand-primary); + background-color: var(--color-sidebar-item-background--current); + font-weight:300; +} + +.sphinx-tabs-tab{ + color: var(--color-brand-primary); + font-weight:300; +} + +.sphinx-tabs-panel { + border: 0; + border-bottom: 1px solid var(--color-sidebar-item-background--hover); + background: var(--color-background-primary); +} + +button.sphinx-tabs-tab:hover { + background-color: var(--color-sidebar-item-background--hover); +} + +/** Custom classes to fix scrolling in tables by decreasing the + font size or breaking certain columns. + Specify the classes in the Markdown file with, for example: + ```{rst-class} break-col-4 min-width-4-8 + ``` +**/ + +table.dec-font-size { + font-size: smaller; +} +table.break-col-1 td.text-left:first-child { + word-break: break-word; +} +table.break-col-4 td.text-left:nth-child(4) { + word-break: break-word; +} +table.min-width-1-15 td.text-left:first-child { + min-width: 15em; +} +table.min-width-4-8 td.text-left:nth-child(4) { + min-width: 8em; +} + +/** Underline for abbreviations **/ + +abbr[title] { + text-decoration: underline solid #cdcdcd; +} + +/** Use the same style for right-details as for left-details **/ +.bottom-of-page .right-details { + font-size: var(--font-size--small); + display: block; +} + +/** Version switcher */ +button.version_select { + color: var(--color-foreground-primary); + background-color: var(--color-toc-background); + padding: 5px 10px; + border: none; +} + +.version_select:hover, .version_select:focus { + background-color: var(--color-sidebar-item-background--hover); +} + +.version_dropdown { + position: relative; + display: inline-block; + text-align: right; + font-size: var(--sidebar-item-font-size); +} + +.available_versions { + display: none; + position: absolute; + right: 0px; + background-color: var(--color-toc-background); + box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); + z-index: 11; +} + +.available_versions a { + color: var(--color-foreground-primary); + padding: 12px 16px; + text-decoration: none; + display: block; +} + +.available_versions a:hover {background-color: var(--color-sidebar-item-background--current)} + +.show {display:block;} + +/** Fix for nested numbered list - the nested list is lettered **/ +ol.arabic ol.arabic { + list-style: lower-alpha; +} + +/** Make expandable sections look like links **/ +details summary { + color: var(--color-link); +} + +/** Fix the styling of the version box for readthedocs **/ + +#furo-readthedocs-versions .rst-versions, #furo-readthedocs-versions .rst-current-version, #furo-readthedocs-versions:focus-within .rst-current-version, #furo-readthedocs-versions:hover .rst-current-version { + background: var(--color-sidebar-item-background--hover); +} + +.rst-versions .rst-other-versions dd a { + color: var(--color-link); +} + +#furo-readthedocs-versions:focus-within .rst-current-version .fa-book, #furo-readthedocs-versions:hover .rst-current-version .fa-book, .rst-versions .rst-other-versions { + color: var(--color-sidebar-link-text); +} + +.rst-versions .rst-current-version { + color: var(--color-version-popup); + font-weight: bolder; +} diff --git a/docs/.sphinx/_static/favicon.png b/docs/.sphinx/_static/favicon.png new file mode 100644 index 00000000..c7109908 Binary files /dev/null and b/docs/.sphinx/_static/favicon.png differ diff --git a/docs/.sphinx/_static/github_issue_links.css b/docs/.sphinx/_static/github_issue_links.css new file mode 100644 index 00000000..af4be86c --- /dev/null +++ b/docs/.sphinx/_static/github_issue_links.css @@ -0,0 +1,24 @@ +.github-issue-link-container { + padding-right: 0.5rem; +} +.github-issue-link { + font-size: var(--font-size--small); + font-weight: bold; + background-color: #DD4814; + padding: 13px 23px; + text-decoration: none; +} +.github-issue-link:link { + color: #FFFFFF; +} +.github-issue-link:visited { + color: #FFFFFF +} +.muted-link.github-issue-link:hover { + color: #FFFFFF; + text-decoration: underline; +} +.github-issue-link:active { + color: #FFFFFF; + text-decoration: underline; +} diff --git a/docs/.sphinx/_static/github_issue_links.js b/docs/.sphinx/_static/github_issue_links.js new file mode 100644 index 00000000..980609cd --- /dev/null +++ b/docs/.sphinx/_static/github_issue_links.js @@ -0,0 +1,26 @@ +window.onload = function() { + const link = document.createElement("a"); + link.classList.add("muted-link"); + link.classList.add("github-issue-link"); + link.text = "Give feedback"; + link.href = ( + github_url + + "/issues/new?" + + "title=docs%3A+TYPE+YOUR+QUESTION+HERE" + + "&body=*Please describe the question or issue you're facing with " + + `"${document.title}"` + + ".*" + + "%0A%0A%0A%0A%0A" + + "---" + + "%0A" + + `*Reported+from%3A+${location.href}*` + ); + link.target = "_blank"; + + const div = document.createElement("div"); + div.classList.add("github-issue-link-container"); + div.append(link) + + const container = document.querySelector(".article-container > .content-icon-container"); + container.prepend(div); +}; diff --git a/docs/.sphinx/_static/header-nav.js b/docs/.sphinx/_static/header-nav.js new file mode 100644 index 00000000..3608576e --- /dev/null +++ b/docs/.sphinx/_static/header-nav.js @@ -0,0 +1,10 @@ +$(document).ready(function() { + $(document).on("click", function () { + $(".more-links-dropdown").hide(); + }); + + $('.nav-more-links').click(function(event) { + $('.more-links-dropdown').toggle(); + event.stopPropagation(); + }); +}) diff --git a/docs/.sphinx/_static/header.css b/docs/.sphinx/_static/header.css new file mode 100644 index 00000000..0b944090 --- /dev/null +++ b/docs/.sphinx/_static/header.css @@ -0,0 +1,167 @@ +.p-navigation { + border-bottom: 1px solid var(--color-sidebar-background-border); +} + +.p-navigation__nav { + background: #333333; + display: flex; +} + +.p-logo { + display: flex !important; + padding-top: 0 !important; + text-decoration: none; +} + +.p-logo-image { + height: 44px; + padding-right: 10px; +} + +.p-logo-text { + margin-top: 18px; + color: white; + text-decoration: none; +} + +ul.p-navigation__links { + display: flex; + list-style: none; + margin-left: 0; + margin-top: auto; + margin-bottom: auto; + max-width: 800px; + width: 100%; +} + +ul.p-navigation__links li { + margin: 0 auto; + text-align: center; + width: 100%; +} + +ul.p-navigation__links li a { + background-color: rgba(0, 0, 0, 0); + border: none; + border-radius: 0; + color: var(--color-sidebar-link-text); + display: block; + font-weight: 400; + line-height: 1.5rem; + margin: 0; + overflow: hidden; + padding: 1rem 0; + position: relative; + text-align: left; + text-overflow: ellipsis; + transition-duration: .1s; + transition-property: background-color, color, opacity; + transition-timing-function: cubic-bezier(0.215, 0.61, 0.355, 1); + white-space: nowrap; + width: 100%; +} + +ul.p-navigation__links .p-navigation__link { + color: #ffffff; + font-weight: 300; + text-align: center; + text-decoration: none; +} + +ul.p-navigation__links .p-navigation__link:hover { + background-color: #2b2b2b; +} + +ul.p-navigation__links .p-dropdown__link:hover { + background-color: var(--color-sidebar-item-background--hover); +} + +ul.p-navigation__links .p-navigation__sub-link { + background: var(--color-background-primary); + padding: .5rem 0 .5rem .5rem; + font-weight: 300; +} + +ul.p-navigation__links .more-links-dropdown li a { + border-left: 1px solid var(--color-sidebar-background-border); + border-right: 1px solid var(--color-sidebar-background-border); +} + +ul.p-navigation__links .more-links-dropdown li:first-child a { + border-top: 1px solid var(--color-sidebar-background-border); +} + +ul.p-navigation__links .more-links-dropdown li:last-child a { + border-bottom: 1px solid var(--color-sidebar-background-border); +} + +ul.p-navigation__links .p-navigation__logo { + padding: 0.5rem; +} + +ul.p-navigation__links .p-navigation__logo img { + width: 40px; +} + +ul.more-links-dropdown { + display: none; + overflow-x: visible; + height: 0; + z-index: 55; + padding: 0; + position: relative; + list-style: none; + margin-bottom: 0; + margin-top: 0; +} + +.nav-more-links::after { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16'%3E%3Cpath fill='%23111' d='M8.187 11.748l6.187-6.187-1.06-1.061-5.127 5.127L3.061 4.5 2 5.561z'/%3E%3C/svg%3E"); + background-position: center; + background-repeat: no-repeat; + background-size: contain; + content: ""; + display: block; + filter: invert(100%); + height: 1rem; + pointer-events: none; + position: absolute; + right: 1rem; + text-indent: calc(100% + 10rem); + top: calc(1rem + 0.25rem); + width: 1rem; +} + +.nav-ubuntu-com { + display: none; +} + +@media only screen and (min-width: 480px) { + ul.p-navigation__links li { + width: 100%; + } + + .nav-ubuntu-com { + display: inherit; + } +} + +@media only screen and (max-width: 800px) { + .nav-more-links { + margin-left: auto !important; + padding-right: 2rem !important; + width: 8rem !important; + } +} + +@media only screen and (min-width: 800px) { + ul.p-navigation__links li { + width: 100% !important; + } +} + +@media only screen and (min-width: 1310px) { + ul.p-navigation__links { + margin-left: calc(50% - 41em); + } +} diff --git a/docs/.sphinx/_static/tag.png b/docs/.sphinx/_static/tag.png new file mode 100644 index 00000000..f6f6e5aa Binary files /dev/null and b/docs/.sphinx/_static/tag.png differ diff --git a/docs/.sphinx/_templates/base.html b/docs/.sphinx/_templates/base.html new file mode 100644 index 00000000..62ffe6b8 --- /dev/null +++ b/docs/.sphinx/_templates/base.html @@ -0,0 +1,7 @@ +{% extends "furo/base.html" %} + +{% block theme_scripts %} + +{% endblock theme_scripts %} diff --git a/docs/.sphinx/_templates/footer.html b/docs/.sphinx/_templates/footer.html new file mode 100644 index 00000000..75944868 --- /dev/null +++ b/docs/.sphinx/_templates/footer.html @@ -0,0 +1,90 @@ +{# ru-fu: copied from Furo, with modifications as stated below #} + + +
+
+ {%- if show_copyright %} + + {%- endif %} + + {# ru-fu: removed "Made with" #} + + {%- if last_updated -%} +
+ {% trans last_updated=last_updated|e -%} + Last updated on {{ last_updated }} + {%- endtrans -%} +
+ {%- endif %} + + {%- if show_source and has_source and sourcename %} + + {%- endif %} +
+
+ + {# ru-fu: replaced RTD icons with our links #} + + {% if discourse %} + + {% endif %} + + {% if github_url and github_version and github_folder %} + + {% if github_issues %} + + {% endif %} + + + {% endif %} + + +
+
+ diff --git a/docs/.sphinx/_templates/header.html b/docs/.sphinx/_templates/header.html new file mode 100644 index 00000000..1a128b6f --- /dev/null +++ b/docs/.sphinx/_templates/header.html @@ -0,0 +1,36 @@ + diff --git a/docs/.sphinx/_templates/page.html b/docs/.sphinx/_templates/page.html new file mode 100644 index 00000000..bda30610 --- /dev/null +++ b/docs/.sphinx/_templates/page.html @@ -0,0 +1,49 @@ +{% extends "furo/page.html" %} + +{% block footer %} + {% include "footer.html" %} +{% endblock footer %} + +{% block body -%} + {% include "header.html" %} + {{ super() }} +{%- endblock body %} + +{% if meta and ((meta.discourse and discourse_prefix) or meta.relatedlinks) %} + {% set furo_hide_toc_orig = furo_hide_toc %} + {% set furo_hide_toc=false %} +{% endif %} + +{% block right_sidebar %} +
+ {% if not furo_hide_toc_orig %} +
+ + {{ _("Contents") }} + +
+
+
+ {{ toc }} +
+
+ {% endif %} + {% if meta and ((meta.discourse and discourse_prefix) or meta.relatedlinks) %} + + + {% endif %} +
+{% endblock right_sidebar %} diff --git a/docs/.sphinx/requirements.txt b/docs/.sphinx/requirements.txt new file mode 100644 index 00000000..0c267c75 --- /dev/null +++ b/docs/.sphinx/requirements.txt @@ -0,0 +1,13 @@ +sphinx==7.1.2 +sphinx-autobuild +sphinx-design +furo +sphinx-tabs +sphinx-reredirects +pyspelling +sphinxext-opengraph +lxd-sphinx-extensions +sphinx-copybutton +myst-parser +sphinxcontrib-jquery +sphinx-notfound-page diff --git a/docs/.sphinx/spellingcheck.yaml b/docs/.sphinx/spellingcheck.yaml new file mode 100644 index 00000000..ebef2081 --- /dev/null +++ b/docs/.sphinx/spellingcheck.yaml @@ -0,0 +1,27 @@ +matrix: +- name: rST files + aspell: + lang: en + d: en_GB + dictionary: + wordlists: + - .wordlist.txt + output: .sphinx/.wordlist.dic + sources: + - _build/**/*.html + pipeline: + - pyspelling.filters.html: + comments: false + attributes: + - title + - alt + ignores: + - code + - pre + - spellexception + - link + - title + - div.relatedlinks + - div.visually-hidden + - img + - a.p-navigation__link diff --git a/docs/.wokeignore b/docs/.wokeignore new file mode 100644 index 00000000..c64a6037 --- /dev/null +++ b/docs/.wokeignore @@ -0,0 +1,4 @@ +# the cheat sheets contain a link to a repository with a block word which we +# cannot avoid for now, ie +# https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html +doc-cheat-sheet* diff --git a/docs/.wordlist.txt b/docs/.wordlist.txt new file mode 100644 index 00000000..5fb54827 --- /dev/null +++ b/docs/.wordlist.txt @@ -0,0 +1,47 @@ +addons +API +APIs +balancer +CharmHub +CLI +DEX +Diátaxis +dropdown +EBS +EKS +favicon +filesystem +Grafana +hostname +IAM +installable +Jira +JSON +Juju +Kubeflow +Kubernetes +lifecycle +Makefile +MicroK +MinIO +MLflow +MLOps +MyST +namespace +namespaces +NodePort +observability +OIDC +OLM +Permalink +ReadMe +readthedocs +reproducibility +reST +reStructuredText +RTD +subdirectories +subtree +UI +TensorBoard +VM diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..dcf383a4 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,65 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build +VENV = .sphinx/venv/bin/activate + + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +install: + @echo "... setting up virtualenv" + python3 -m venv .sphinx/venv + . $(VENV); pip install --upgrade -r .sphinx/requirements.txt + + @echo "\n" \ + "--------------------------------------------------------------- \n" \ + "* watch, build and serve the documentation: make run \n" \ + "* only build: make html \n" \ + "* only serve: make serve \n" \ + "* clean built doc files: make clean-doc \n" \ + "* clean full environment: make clean \n" \ + "* check spelling: make spelling \n" \ + "* check inclusive language: make woke \n" \ + "--------------------------------------------------------------- \n" +run: + . $(VENV); sphinx-autobuild -c . -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" + +html: + . $(VENV); $(SPHINXBUILD) -c . -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" -w .sphinx/warnings.txt + +epub: + . $(VENV); $(SPHINXBUILD) -c . -b epub "$(SOURCEDIR)" "$(BUILDDIR)" -w .sphinx/warnings.txt + +serve: + cd "$(BUILDDIR)"; python3 -m http.server 8000 + +clean: clean-doc + rm -rf .sphinx/venv + +clean-doc: + git clean -fx "$(BUILDDIR)" + +spelling: html + . $(VENV) ; python3 -m pyspelling -c .sphinx/spellingcheck.yaml + +linkcheck: + . $(VENV) ; $(SPHINXBUILD) -c . -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)" + +woke: + type woke >/dev/null 2>&1 || { sudo snap install woke; } + woke *.rst **/*.rst -c https://github.com/canonical-web-and-design/Inclusive-naming/raw/main/config.yml + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + . $(VENV); $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..be11d82f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,182 @@ +import sys + +sys.path.append('./') +from custom_conf import * + +# Configuration file for the Sphinx documentation builder. +# You should not do any modifications to this file. Put your custom +# configuration into the custom_conf.py file. +# If you need to change this file, contribute the changes upstream. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +############################################################ +### Extensions +############################################################ + +extensions = [ + 'sphinx_design', + 'sphinx_tabs.tabs', + 'sphinx_reredirects', + 'youtube-links', + 'related-links', + 'custom-rst-roles', + 'terminal-output', + 'sphinx_copybutton', + 'sphinxext.opengraph', + 'myst_parser', + 'sphinxcontrib.jquery', + 'notfound.extension' +] +extensions.extend(custom_extensions) + +### Configuration for extensions + +# Additional MyST syntax +myst_enable_extensions = [ + 'substitution', + 'deflist' +] + +# Used for related links +if 'discourse' in html_context: + html_context['discourse_prefix'] = html_context['discourse'] + '/t/' + +# The default for notfound_urls_prefix usually works, but not for +# documentation on documentation.ubuntu.com +if slug: + notfound_urls_prefix = '/' + slug + '/en/latest/' + +notfound_context = { + 'title': 'Page not found', + 'body': '

Page not found

\n\n

Sorry, but the documentation page that you are looking for was not found.

\n

Documentation changes over time, and pages are moved around. We try to redirect you to the updated content where possible, but unfortunately, that didn\'t work this time (maybe because the content you were looking for does not exist in this version of the documentation).

\n

You can try to use the navigation to locate the content you\'re looking for, or search for a similar page.

\n', +} + +# Default image for OGP (to prevent font errors, see +# https://github.com/canonical/sphinx-docs-starter-pack/pull/54 ) +if not 'ogp_image' in locals(): + ogp_image = 'https://assets.ubuntu.com/v1/253da317-image-document-ubuntudocs.svg' + +############################################################ +### General configuration +############################################################ + +exclude_patterns = [ + '_build', + 'Thumbs.db', + '.DS_Store', + '.sphinx', + 'doc-cheat-sheet*', +] +exclude_patterns.extend(custom_excludes) + +rst_epilog = ''' +.. include:: /reuse/links.txt +''' +if 'custom_rst_epilog' in locals(): + rst_epilog = custom_rst_epilog + +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + +if not 'conf_py_path' in html_context and 'github_folder' in html_context: + html_context['conf_py_path'] = html_context['github_folder'] + +# For ignoring specific links +linkcheck_anchors_ignore_for_url = [ + r'https://github\.com/.*' +] + +############################################################ +### Styling +############################################################ + +# Find the current builder +builder = 'dirhtml' +if '-b' in sys.argv: + builder = sys.argv[sys.argv.index('-b')+1] + +# Setting templates_path for epub makes the build fail +if builder == 'dirhtml' or builder == 'html': + templates_path = ['.sphinx/_templates'] + +# Theme configuration +html_theme = 'furo' +html_last_updated_fmt = '' +html_permalinks_icon = '¶' +html_theme_options = { + 'light_css_variables': { + 'font-stack': 'Ubuntu, -apple-system, Segoe UI, Roboto, Oxygen, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif', + 'font-stack--monospace': 'Ubuntu Mono, Consolas, Monaco, Courier, monospace', + 'color-foreground-primary': '#111', + 'color-foreground-secondary': 'var(--color-foreground-primary)', + 'color-foreground-muted': '#333', + 'color-background-secondary': '#FFF', + 'color-background-hover': '#f2f2f2', + 'color-brand-primary': '#111', + 'color-brand-content': '#06C', + 'color-api-background': '#cdcdcd', + 'color-inline-code-background': 'rgba(0,0,0,.03)', + 'color-sidebar-link-text': '#111', + 'color-sidebar-item-background--current': '#ebebeb', + 'color-sidebar-item-background--hover': '#f2f2f2', + 'toc-font-size': 'var(--font-size--small)', + 'color-admonition-title-background--note': 'var(--color-background-primary)', + 'color-admonition-title-background--tip': 'var(--color-background-primary)', + 'color-admonition-title-background--important': 'var(--color-background-primary)', + 'color-admonition-title-background--caution': 'var(--color-background-primary)', + 'color-admonition-title--note': '#24598F', + 'color-admonition-title--tip': '#24598F', + 'color-admonition-title--important': '#C7162B', + 'color-admonition-title--caution': '#F99B11', + 'color-highlighted-background': '#EbEbEb', + 'color-link-underline': 'var(--color-background-primary)', + 'color-link-underline--hover': 'var(--color-background-primary)', + 'color-version-popup': '#772953' + }, + 'dark_css_variables': { + 'color-foreground-secondary': 'var(--color-foreground-primary)', + 'color-foreground-muted': '#CDCDCD', + 'color-background-secondary': 'var(--color-background-primary)', + 'color-background-hover': '#666', + 'color-brand-primary': '#fff', + 'color-brand-content': '#06C', + 'color-sidebar-link-text': '#f7f7f7', + 'color-sidebar-item-background--current': '#666', + 'color-sidebar-item-background--hover': '#333', + 'color-admonition-background': 'transparent', + 'color-admonition-title-background--note': 'var(--color-background-primary)', + 'color-admonition-title-background--tip': 'var(--color-background-primary)', + 'color-admonition-title-background--important': 'var(--color-background-primary)', + 'color-admonition-title-background--caution': 'var(--color-background-primary)', + 'color-admonition-title--note': '#24598F', + 'color-admonition-title--tip': '#24598F', + 'color-admonition-title--important': '#C7162B', + 'color-admonition-title--caution': '#F99B11', + 'color-highlighted-background': '#666', + 'color-link-underline': 'var(--color-background-primary)', + 'color-link-underline--hover': 'var(--color-background-primary)', + 'color-version-popup': '#F29879' + }, +} + +############################################################ +### Additional files +############################################################ + +html_static_path = ['.sphinx/_static'] + +html_css_files = [ + 'custom.css', + 'header.css', + 'github_issue_links.css', +] +html_css_files.extend(custom_html_css_files) + +html_js_files = ['header-nav.js'] +if 'github_issues' in html_context and html_context['github_issues'] and not disable_feedback_button: + html_js_files.append('github_issue_links.js') +html_js_files.extend(custom_html_js_files) diff --git a/docs/custom_conf.py b/docs/custom_conf.py new file mode 100644 index 00000000..20bb6e1d --- /dev/null +++ b/docs/custom_conf.py @@ -0,0 +1,126 @@ +import datetime + +# Custom configuration for the Sphinx documentation builder. +# All configuration specific to your project should be done in this file. +# +# The file is included in the common conf.py configuration file. +# You can modify any of the settings below or add any configuration that +# is not covered by the common conf.py file. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +############################################################ +### Project information +############################################################ + +# Product name +project = 'Charmed MLflow' +author = 'Canonical Group Ltd' + +# Uncomment if your product uses release numbers +# release = '1.0' + +# The default value uses the current year as the copyright year +copyright = '%s, %s' % (datetime.date.today().year, author) + +## Open Graph configuration - defines what is displayed in the website preview +# The URL of the documentation output +ogp_site_url = 'https://canonical-starter-pack.readthedocs-hosted.com/' +# The documentation website name (usually the same as the product name) +ogp_site_name = 'mlflow' +# An image or logo that is used in the preview +ogp_image = 'https://assets.ubuntu.com/v1/253da317-image-document-ubuntudocs.svg' + +# Update with the favicon for your product (default is the circle of friends) +html_favicon = '.sphinx/_static/favicon.png' + +# (Some settings must be part of the html_context dictionary, while others +# are on root level. Don't move the settings.) +html_context = { + + # Change to the link to your product website (without "https://") + 'product_page': 'documentation.ubuntu.com', + + # Add your product tag to ".sphinx/_static" and change the path + # here (start with "_static"), default is the circle of friends + 'product_tag': '_static/tag.png', + + # Change to the discourse instance you want to be able to link to + # using the :discourse: metadata at the top of a file + # (use an empty value if you don't want to link) + 'discourse': 'https://discourse.charmhub.io/tag/mlflow', + + # Change to the GitHub info for your project + 'github_url': 'https://github.com/canonical/mlflow-operator', + + # Change to the branch for this version of the documentation + 'github_version': 'main', + + # Change to the folder that contains the documentation + # (usually "/" or "/docs/") + 'github_folder': '/docs/', + + # Change to an empty value if your GitHub repo doesn't have issues enabled. + # This will disable the feedback button and the issue link in the footer. + 'github_issues': 'enabled' +} + +# If your project is on documentation.ubuntu.com, specify the project +# slug (for example, "lxd") here. +slug = "" + +############################################################ +### Redirects +############################################################ + +# Set up redirects (https://documatt.gitlab.io/sphinx-reredirects/usage.html) +# For example: 'explanation/old-name.html': '../how-to/prettify.html', + +redirects = {} + +############################################################ +### Link checker exceptions +############################################################ + +# Links to ignore when checking links + +linkcheck_ignore = [ + 'http://127.0.0.1:8000' + ] + +############################################################ +### Additions to default configuration +############################################################ + +## The following settings are appended to the default configuration. +## Use them to extend the default functionality. + +# Add extensions +custom_extensions = [] + +# Add files or directories that should be excluded from processing. +custom_excludes = [] + +# Add CSS files (located in .sphinx/_static/) +custom_html_css_files = [] + +# Add JavaScript files (located in .sphinx/_static/) +custom_html_js_files = [] + +## The following settings override the default configuration. + +# Specify a reST string that is included at the end of each file. +# If commented out, use the default (which pulls the reuse/links.txt +# file into each reST file). +# custom_rst_epilog = '' + +# By default, the documentation includes a feedback button at the top. +# You can disable it by setting the following configuration to True. +disable_feedback_button = False + +############################################################ +### Additional configuration +############################################################ + +## Add any configuration that is not covered by the common conf.py file. diff --git a/docs/doc-cheat-sheet-myst.md b/docs/doc-cheat-sheet-myst.md new file mode 100644 index 00000000..19b07f53 --- /dev/null +++ b/docs/doc-cheat-sheet-myst.md @@ -0,0 +1,243 @@ +--- +orphan: true +myst: + substitutions: + reuse_key: "This is **included** text." + advanced_reuse_key: "This is a substitution that includes a code block: + ``` + code block + ```" +--- + +(cheat-sheet-myst)= +# Markdown/MyST cheat sheet + +This file contains the syntax for commonly used Markdown and MyST markup. +Open it in your text editor to quickly copy and paste the markup you need. + +Also see the [MyST documentation](https://myst-parser.readthedocs.io/en/latest/index.html) for detailed information, and the [Canonical Documentation Style Guide](https://docs.ubuntu.com/styleguide/en) for general style conventions. + +## H2 heading + +### H3 heading + +#### H4 heading + +##### H5 heading + +## Inline formatting + +- {guilabel}`UI element` +- `code` +- {command}`command` +- {kbd}`Key` +- *Italic* +- **Bold** + +## Code blocks + +Start a code block: + + code: + - example: true + +``` +# Demonstrate a code block +code: + - example: true +``` + +```yaml +# Demonstrate a code block +code: + - example: true +``` + +(_a_section_target)= +## Links + +- [Canonical website](https://canonical.com/) +- https://canonical.com/ +- {ref}`a_section_target` +- {ref}`Link text ` +- {doc}`index` +- {doc}`Link text ` + + +## Navigation + +Use the following syntax:: + + ```{toctree} + :hidden: + + sub-page1 + sub-page2 + ``` + +## Lists + +1. Step 1 + - Item 1 + * Sub-item + - Item 2 + 1. Sub-step 1 + 1. Sub-step 2 +1. Step 2 + 1. Sub-step 1 + - Item + 1. Sub-step 2 + +Term 1 +: Definition + +Term 2 +: Definition + +## Tables + +## Markdown tables + +| Header 1 | Header 2 | +|------------------------------------|----------| +| Cell 1
Second paragraph | Cell 2 | +| Cell 3 | Cell 4 | + +Centred: + +| Header 1 | Header 2 | +|:----------------------------------:|:--------:| +| Cell 1
Second paragraph | Cell 2 | +| Cell 3 | Cell 4 | + +## List tables + +```{list-table} + :header-rows: 1 + +* - Header 1 + - Header 2 +* - Cell 1 + + Second paragraph + - Cell 2 +* - Cell 3 + - Cell 4 +``` + +Centred: + +```{list-table} + :header-rows: 1 + :align: center + +* - Header 1 + - Header 2 +* - Cell 1 + + Second paragraph + - Cell 2 +* - Cell 3 + - Cell 4 +``` + +## Notes + +```{note} +A note. +``` + +```{tip} +A tip. +``` + +```{important} +Important information +``` + +```{caution} +This might damage your hardware! +``` + +## Images + +![Alt text](https://assets.ubuntu.com/v1/b3b72cb2-canonical-logo-166.png) + +```{figure} https://assets.ubuntu.com/v1/b3b72cb2-canonical-logo-166.png + :width: 100px + :alt: Alt text + + Figure caption +``` + +## Reuse + +### Keys + +Keys can be defined at the top of a file, or in a `myst_substitutions` option in `conf.py`. + +{{reuse_key}} + +{{advanced_reuse_key}} + +### File inclusion + +```{include} index.rst + :start-after: include_start + :end-before: include_end +``` + +## Tabs + +````{tabs} +```{group-tab} Tab 1 + +Content Tab 1 +``` + +```{group-tab} Tab 2 +Content Tab 2 +``` +```` + +## Glossary + +```{glossary} + +some term + Definition of the example term. +``` + +{term}`some term` + +## More useful markup + +- ```{versionadded} X.Y +- {abbr}`API (Application Programming Interface)` + +---- + +## Custom extensions + +Related links at the top of the page (surrounded by `---`): + + relatedlinks: https://github.com/canonical/lxd-sphinx-extensions, [RTFM](https://www.google.com) + discourse: 12345 + +Terms that should not be checked by the spelling checker: {spellexception}`PurposelyWrong` + +A terminal view with input and output: + +```{terminal} + :input: command + :user: root + :host: vampyr + +the output +``` + +A link to a YouTube video: + +```{youtube} https://www.youtube.com/watch?v=iMLiK1fX4I0 + :title: Demo +``` diff --git a/docs/doc-cheat-sheet.rst b/docs/doc-cheat-sheet.rst new file mode 100644 index 00000000..b12ebcdf --- /dev/null +++ b/docs/doc-cheat-sheet.rst @@ -0,0 +1,258 @@ +:orphan: + +.. _cheat-sheet: + +reStructuredText cheat sheet +============================ + +This file contains the syntax for commonly used reST markup. +Open it in your text editor to quickly copy and paste the markup you need. + +See the `reStructuredText style guide `_ for detailed information and conventions. + +Also see the `Sphinx reStructuredText Primer `_ for more details on reST, and the `Canonical Documentation Style Guide `_ for general style conventions. + +H2 heading +---------- + +H3 heading +~~~~~~~~~~ + +H4 heading +^^^^^^^^^^ + +H5 heading +.......... + +Inline formatting +----------------- + +- :guilabel:`UI element` +- ``code`` +- :file:`file path` +- :command:`command` +- :kbd:`Key` +- *Italic* +- **Bold** + +Code blocks +----------- + +Start a code block:: + + code: + - example: true + +.. code:: + + # Demonstrate a code block + code: + - example: true + +.. code:: yaml + + # Demonstrate a code block + code: + - example: true + +.. _a_section_target: + +Links +----- + +- `Canonical website `_ +- `Canonical website`_ (defined in ``reuse/links.txt`` or at the bottom of the page) +- https:\ //canonical.com/ +- :ref:`a_section_target` +- :ref:`Link text ` +- :doc:`index` +- :doc:`Link text ` + + +Navigation +---------- + +Use the following syntax:: + + .. toctree:: + :hidden: + + sub-page1 + sub-page2 + + +Lists +----- + +1. Step 1 + + - Item 1 + + * Sub-item + - Item 2 + + i. Sub-step 1 + #. Sub-step 2 +#. Step 2 + + a. Sub-step 1 + + - Item + #. Sub-step 2 + +Term 1: + Definition +Term 2: + Definition + +Tables +------ + ++----------------------+------------+ +| Header 1 | Header 2 | ++======================+============+ +| Cell 1 | Cell 2 | +| | | +| Second paragraph | | ++----------------------+------------+ +| Cell 3 | Cell 4 | ++----------------------+------------+ + +.. list-table:: + :header-rows: 1 + + * - Header 1 + - Header 2 + * - Cell 1 + + Second paragraph + - Cell 2 + * - Cell 3 + - Cell 4 + +.. rst-class:: align-center + + +----------------------+------------+ + | Header 1 | Header 2 | + +======================+============+ + | Cell 1 | Cell 2 | + | | | + | Second paragraph | | + +----------------------+------------+ + | Cell 3 | Cell 4 | + +----------------------+------------+ + +.. list-table:: + :header-rows: 1 + :align: center + + * - Header 1 + - Header 2 + * - Cell 1 + + Second paragraph + - Cell 2 + * - Cell 3 + - Cell 4 + +Notes +----- + +.. note:: + A note. + +.. tip:: + A tip. + +.. important:: + Important information + +.. caution:: + This might damage your hardware! + +Images +------ + +.. image:: https://assets.ubuntu.com/v1/b3b72cb2-canonical-logo-166.png + +.. figure:: https://assets.ubuntu.com/v1/b3b72cb2-canonical-logo-166.png + :width: 100px + :alt: Alt text + + Figure caption + +Reuse +----- + +.. |reuse_key| replace:: This is **included** text. + +|reuse_key| + +.. include:: index.rst + :start-after: include_start + :end-before: include_end + +Tabs +---- + +.. tabs:: + + .. group-tab:: Tab 1 + + Content Tab 1 + + .. group-tab:: Tab 2 + + Content Tab 2 + + +Glossary +-------- + +.. glossary:: + + example term + Definition of the example term. + +:term:`example term` + +More useful markup +------------------ + +- .. versionadded:: X.Y +- | Line 1 + | Line 2 + | Line 3 +- .. This is a comment +- :abbr:`API (Application Programming Interface)` + +---- + +Custom extensions +----------------- + +Related links at the top of the page:: + + :relatedlinks: https://github.com/canonical/lxd-sphinx-extensions, [RTFM](https://www.google.com) + :discourse: 12345 + +Terms that should not be checked by the spelling checker: :spellexception:`PurposelyWrong` + +A terminal view with input and output: + +.. terminal:: + :input: command + :user: root + :host: vampyr + + the output + +A link to a YouTube video: + +.. youtube:: https://www.youtube.com/watch?v=iMLiK1fX4I0 + :title: Demo + + + +.. LINKS +.. _Canonical website: https://canonical.com/ diff --git a/docs/explanation/index.rst b/docs/explanation/index.rst new file mode 100644 index 00000000..9737e91b --- /dev/null +++ b/docs/explanation/index.rst @@ -0,0 +1,4 @@ +Explanation +=========== + +Coming soon. \ No newline at end of file diff --git a/docs/how-to/create-ck8s-aws.rst b/docs/how-to/create-ck8s-aws.rst new file mode 100644 index 00000000..15a710a6 --- /dev/null +++ b/docs/how-to/create-ck8s-aws.rst @@ -0,0 +1,87 @@ +Create an MLOps-ready Charmed Kubernetes cluster +================================================ + +This how-to guide will show you how to create a Charmed Kubernetes (CK8s) cluster with an appropriate configuration for deploying an MLOps platforms such as Kubeflow or MLflow. + +**Prerequisites** + +- A local machine with Ubuntu 22.04 or later. +- An AWS account (`How to create an AWS account `_). + +Install and set up AWS CLI +--------------------------- + +First, `install the AWS CLI `_ on your local machine, and then set it up. You can use any of the authentication methods available for the AWS CLI. For example, you can use `IAM user credentials `_. + +Install other tools +------------------- + +To install some helpful tools, run this command: + +.. code-block:: bash + + sudo snap install juju --classic --channel=2.9/stable + for snap in juju-wait kubectl jq; \ + do sudo snap install $snap --classic; \ + done + +This installs the following: + +* ``juju``: Needed to deploy and manage the CK8s cluster. +* ``juju-wait``: CLI tool used for waiting during Juju deployments. +* ``kubectl``: Kubernetes client used to communicate with a Kubernetes cluster. +* ``jq``: A lightweight and versatile command-line tool for parsing and manipulating JSON data. + + + +Setup Juju with AWS +------------------- + +Set up Juju to communicate with AWS. + +.. code-block:: bash + + juju add-credential aws + +You will be prompted for information related to your AWS account that you provided while setting up the AWS CLI (e.g., access key, secret access key). + +Create Juju controller +---------------------- + +Bootstrap a Juju controller that will be responsible for deploying cluster applications. + +.. code-block:: bash + + juju bootstrap aws kf-controller + +Deploy Charmed Kubernetes 1.24 +------------------------------ + +Clone the `Charmed Kubernetes bundle repository `_, and update CPU, disk, and memory constraints to meet Kubeflow requirements. + +.. code-block:: bash + + git clone https://github.com/charmed-kubernetes/bundle + sed -i '/^ *charm: kubernetes-worker/,/^ *[^:]*:/s/constraints: cores=2 mem=8G root-disk=16G/constraints: cores=8 mem=32G root-disk=200G/' ./bundle/releases/1.24/bundle.yaml + +Deploy the Charmed Kubernetes bundle on AWS with the storage overlay. This overlay enables you to create Kubernetes volumes backed by AWS EBS. + +.. code-block:: bash + + juju deploy ./bundle/releases/1.24/bundle.yaml \ + --overlay ./bundle/overlays/aws-storage-overlay.yaml --trust + +Wait until all components are ready. + +.. code-block:: bash + + juju-wait -m default -t 3600 + +Retrieve the Kubernetes configuration from the control plane leader unit. + +.. code-block:: bash + + mkdir ~/.kube + juju ssh kubernetes-control-plane/leader -- cat config > ~/.kube/config + +Now you can use ``kubectl`` to talk to your newly created Charmed Kubernetes cluster. \ No newline at end of file diff --git a/docs/how-to/deploy-ck8s-aws.rst b/docs/how-to/deploy-ck8s-aws.rst new file mode 100644 index 00000000..eba1aed5 --- /dev/null +++ b/docs/how-to/deploy-ck8s-aws.rst @@ -0,0 +1,70 @@ +Deploy Charmed MLflow to Charmed Kubernetes on AWS +======================================================== + ++-----------+---------+ +| Component | Version | ++-----------+---------+ +| MLflow | 2 | ++-----------+---------+ + +This guide shows how to connect Juju to an existing `Charmed Kubernetes `_ (CK8s) cluster and deploy the MLflow bundle on top of it. + +Prerequisites +------------- + +We assume that you have access to a CK8s cluster using ``kubectl``. If you don't have a cluster set up, you can follow this guide: :doc:`Create CK8s on AWS `. + +Install Juju +------------ + +Install Juju: + +.. code-block:: bash + + sudo snap install juju --classic --channel=2.9/stable + +Connect Juju to Charmed Kubernetes cluster +------------------------------------------ + +Configure Juju to communicate with the CK8s cluster by creating a controller: + +.. code-block:: bash + + juju add-k8s charmed-k8s-aws --controller $(juju switch | cut -d: -f1) \ + --storage=cdk-ebs + +Create a model. The model name is up to you. However, if you plan to connect MLflow with Kubeflow you must use ``kubeflow`` as the model name. + +.. code-block:: bash + + juju add-model kubeflow charmed-k8s-aws + +Deploy MLflow bundle +-------------------- + +Deploy the MLflow bundle: + +.. code-block:: bash + + juju deploy mlflow --channel=2.1/stable --trust + +Wait until the deployments are active: + +.. code-block:: bash + + juju-wait -m kubeflow -t 2700 + +Connect to MLflow dashboard +--------------------------- + +By default, the MLflow UI is exposed as a NodePort Kubernetes service, accessible at each node's IP address. MLflow runs on port 31380 by default. AWS nodes are EC2 instances. To connect to an instance, it must be configured to allow traffic to this port. + +You can connect to any EC2 instance in the cluster. List all available nodes in your Kubernetes cluster and choose any ``EXTERNAL-IP`` that you will use to access the MLflow UI: + +.. code-block:: bash + + kubectl get nodes -o wide + +In your AWS account find the EC2 instance with that particular ``EXTERNAL-IP`` and enable access to the port 31380 in the inbound rules of the security group. To see how, consult `AWS docs `_. + +Open a web browser and visit ``:31380`` to access the MLflow UI. diff --git a/docs/how-to/deploy-eks.rst b/docs/how-to/deploy-eks.rst new file mode 100644 index 00000000..80a2ddd4 --- /dev/null +++ b/docs/how-to/deploy-eks.rst @@ -0,0 +1,68 @@ +Deploy Charmed MLflow to EKS +============================ + ++-----------+---------+ +| Component | Version | ++-----------+---------+ +| MLflow | 2 | ++-----------+---------+ + +This guide shows how to deploy Charmed MLflow on `AWS Elastic Kubernetes Service `_ (EKS). In this guide, we will create an AWS EKS cluster, connect Juju to it, and deploy the MLflow bundle. + +Prerequisites: +-------------- +We assume the following: + +- Your machine runs Ubuntu 22.04 or later +- You have an AWS account (`How to create an AWS account `_) + +Create EKS cluster +------------------- +See the `EKS creation guide `_ for how to do that. + +Setup Juju +---------- + +Set up your local ``juju`` to talk to the remote Kubernetes (K8s) cloud. First, install ``juju``: + +.. code-block:: bash + + sudo snap install juju --classic + +Connect Juju to Kubernetes: + +.. code-block:: bash + + juju add-k8s kubeflow + +.. note:: You must choose the name ``kubeflow`` if you plan to connect MLflow to Kubeflow. Otherwise you can choose any name. + +Create a controller: + +.. code-block:: bash + + juju bootstrap --no-gui kubeflow kubeflow-controller + +.. note:: You can use whatever controller name you like here, we chose ``kubeflow-controller``. + +Add a Juju model: + +.. code-block:: bash + + juju add-model kubeflow + +.. note:: You must choose the name ``kubeflow`` if you plan to connect MLflow to Kubeflow. Otherwise you can choose any name. + +Deploy MLflow bundle +--------------------- +Deploy the MLflow bundle with the following command: + +.. code-block:: bash + + juju deploy mlflow --channel=2.1/stable --trust + +Wait until all charms are in the active state. You can check the state of the charms with the command: + +.. code-block:: bash + + juju status --watch 5s --relations diff --git a/docs/how-to/deploy-mlflow-kubeflow-eks.rst b/docs/how-to/deploy-mlflow-kubeflow-eks.rst new file mode 100644 index 00000000..bf280e40 --- /dev/null +++ b/docs/how-to/deploy-mlflow-kubeflow-eks.rst @@ -0,0 +1,130 @@ +Deploy Charmed MLflow and Kubeflow to EKS +========================================= + ++-----------+---------+ +| Component | Version | ++-----------+---------+ +| MLflow | 2 | ++-----------+---------+ + +This guide shows how to deploy Charmed MLflow alongside Kubeflow on `AWS Elastic Kubernetes Service `_ (EKS). In this guide, we will create an AWS EKS cluster, connect Juju to it, deploy the MLflow and Kubeflow bundles, and relate them to each other. + +Prerequisites +------------- + +We assume the following: + +- Your machine runs Ubuntu 22.04 or later +- You have an AWS account (`How to create an AWS account `_) + +Deploy EKS cluster +------------------- + +See our `EKS creation guide `_ for a complete guide on how to do this. **Do not forget** to edit the ``instanceType`` field under ``managedNodeGroups[0].instanceType`` from ``t2.2xlarge`` to ``t3.2xlarge``, as instructed in the guide, since worker nodes of type ``t3.2xlarge`` are required for deploying both MLflow and Kubeflow. + +Setup Juju +---------- + +Set up your local ``juju`` to talk to the remote Kubernetes cloud. First, install Juju with: + +.. code-block:: bash + + sudo snap install juju --classic + +Connect it to Kubernetes: + +.. code-block:: bash + + juju add-k8s kubeflow + +Create the controller: + +.. code-block:: bash + + juju bootstrap --no-gui kubeflow kubeflow-controller + +.. note:: we chose the name ``kubeflow-controller``, but you can choose any other name. + +Add a Juju model: + +.. code-block:: bash + + juju add-model kubeflow + +Deploy MLflow bundle +--------------------- + +Deploy the MLflow bundle with the following command: + +.. code-block:: bash + + juju deploy mlflow --channel=2.1/stable --trust + +Wait until all charms are in the active state. You can check the state of the charms with the command: + +.. code-block:: bash + + juju status --watch 5s --relations + +Deploy Kubeflow bundle +---------------------- + +Deploy the Kubeflow bundle with the following command: + +.. code-block:: bash + + juju deploy kubeflow --channel=1.7/stable --trust + +Wait until all charms are in the active state. You can check the state of the charms with the command: + +.. code-block:: bash + + juju status --watch 5s --relations + +Relate MLflow to Kubeflow +------------------------- + +The resource dispatcher is used to connect MLflow with Kubeflow. In particular, it is responsible for configuring MLflow related Kubernetes objects for Kubeflow user namespaces. Deploy the resource dispatcher to the cluster with the command: + +.. code-block:: bash + + juju deploy resource-dispatcher --channel 1.0/stable --trust + +Relate the resource dispatcher to MLflow with the following commands: + +.. code-block:: bash + + juju relate mlflow-server:secrets resource-dispatcher:secrets + juju relate mlflow-server:pod-defaults resource-dispatcher:pod-defaults + +Wait until all charms are in the active state. You can check the state of the charms with the command: + +.. code-block:: bash + + juju status --watch 5s --relations + +Configure Kubeflow dashboard +---------------------------- + +Get the hostname from the ``istio-ingressgateway-workload`` Kubernetes load balancer service: + +.. code-block:: bash + + export INGRESS_HOST=$(kubectl get svc -n kubeflow istio-ingressgateway-workload -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + +Then, configure OIDC and DEX with the ``INGRESS_HOST`` we just retrieved, and also a username and password of your choosing: + +.. code-block:: bash + + juju config dex-auth public-url="http://${INGRESS_HOST}" + juju config oidc-gatekeeper public-url="http://${INGRESS_HOST}" + juju config dex-auth static-password=user123 + juju config dex-auth static-username=user123@email.com + +Wait until all charms are in the active state. You can check the state of the charms with the command: + +.. code-block:: bash + + juju status --watch 5s --relations + +Now you can access the Kubeflow dashboard at the value from ``INGRESS_HOST`` in your browser. \ No newline at end of file diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst new file mode 100644 index 00000000..1dd506bb --- /dev/null +++ b/docs/how-to/index.rst @@ -0,0 +1,39 @@ +How-to guides +============= + +These guides provide practical instructions for specific tasks related to deploying, managing and using MLflow. + +Preparation +----------- + +.. toctree:: + :maxdepth: 1 + + create-ck8s-aws + +Deployment +---------- + +.. toctree:: + :maxdepth: 1 + + deploy-ck8s-aws + deploy-eks + deploy-mlflow-kubeflow-eks + +Integration +----------- + +.. toctree:: + :maxdepth: 1 + + integrate-cos + integrate-ml-ckf-ck8s + +Upgrading +--------- + +.. toctree:: + :maxdepth: 1 + + migrate-v1-v2 \ No newline at end of file diff --git a/docs/how-to/integrate-cos.rst b/docs/how-to/integrate-cos.rst new file mode 100644 index 00000000..5c9acdaf --- /dev/null +++ b/docs/how-to/integrate-cos.rst @@ -0,0 +1,88 @@ +Integrate MLflow with the Canonical Observability Stack (COS) +============================================================= + +This guide shows how to integrate MLflow with the Canonical Observability Stack (COS). + +Prerequisites +------------- + +This guide assumes: + +#. You have deployed the COS stack in the ``cos`` model. For steps on how to do this, see the `MicroK8s tutorial `_. +#. You have deployed the MLflow bundle in the ``kubeflow`` model. For steps on how to do this, see :doc:`../tutorial/mlflow`. + +Deploy Grafana Agent +-------------------- + +Deploy the `Grafana Agent `_ to your ``kubeflow`` model alongside the MLflow bundle. Run the following command: + +.. code-block:: bash + + juju deploy grafana-agent-k8s --channel=edge --trust + +Relate MLflow Server Prometheus Metrics to Grafana Agent +-------------------------------------------------------- + +Establish the relationship between the MLflow Server Prometheus metrics and the Grafana Agent. Use the following command: + +.. code-block:: bash + + juju add-relation mlflow-server:metrics-endpoint grafana-agent-k8s:metrics-endpoint + +Relate Grafana Agent to Prometheus in the COS Model +--------------------------------------------------- + +Next, relate the Grafana Agent to Prometheus in the ``cos`` model. Execute the following command: + +.. code-block:: bash + + juju add-relation grafana-agent-k8s admin/cos.prometheus-receive-remote-write + +Relate MLflow Server in the Kubeflow Model to Grafana Charm in the COS Model +---------------------------------------------------------------------------- + +Establish the relationship between the MLflow Server in the ``kubeflow`` model and the Grafana charm in the ``cos`` model. Run the following command: + +.. code-block:: bash + + juju add-relation mlflow-server admin/cos.grafana-dashboards + +Obtain the Grafana Dashboard Admin Password +------------------------------------------- + +Switch the model to ``cos`` and retrieve the Grafana dashboard admin password. Execute the following commands: + +.. code-block:: bash + + juju switch cos + juju run-action grafana/0 get-admin-password --wait + +Obtain the Grafana Dashboard URL +-------------------------------- + +To access the Grafana dashboard, you need the URL. Run the following command to get the URLs for the COS endpoints: + +.. code-block:: bash + + juju show-unit catalogue/0 | grep url + +You will see a list of endpoints similar to the following: + +.. code-block:: bash + + url: http://10.43.8.34:80/cos-catalogue + url: http://10.43.8.34/cos-grafana + url: http://10.43.8.34:80/cos-prometheus-0 + url: http://10.43.8.34:80/cos-alertmanager + +Choose the ``cos-grafana`` URL and access it in your browser. + +Login to Grafana +---------------- + +Login to Grafana with the password obtained from the previous section. The username is ``admin``. + +Access the dashboard in the UI +------------------------------ + +Go to the left sidebar and choose the MLflow Dashboards from the list. From the General dashboards folder choose the ``MLflow metrics Dashboard``. When accessing the dashboard for the first time, choose some reasonable time range from the top right dropdown. diff --git a/docs/how-to/integrate-ml-ckf-ck8s.rst b/docs/how-to/integrate-ml-ckf-ck8s.rst new file mode 100644 index 00000000..7d2ed589 --- /dev/null +++ b/docs/how-to/integrate-ml-ckf-ck8s.rst @@ -0,0 +1,42 @@ +Integrate Charmed MLflow with Charmed Kubeflow on Charmed Kubernetes +========================================================================= + ++-----------+---------+ +| Component | Version | ++-----------+---------+ +| MLflow | 2 | ++-----------+---------+ + +In this guide, we will guide you through the process of integrating Charmed MLflow with Charmed Kubeflow on `Charmed Kubernetes `_. + +Prerequisites +-------------- +We assume that: + +* You have access to a Charmed Kubernetes cluster using ``kubectl``. If you don't have a cluster set up, you can follow the :doc:`creation guide ` to deploy one on AWS. +* You have deployed the Charmed Kubeflow bundle. If you don't have it, here is `a guide `_ on how to do it. +* You have deployed the Charmed MLflow bundle. To see how, follow our :doc:`deployment guide `. + +Deploy resource dispatcher +-------------------------- + +Deploy the resource dispatcher: + +.. code-block:: bash + + juju deploy resource-dispatcher --channel 1.0/stable --trust + +Relate Resource dispatcher to MLflow +------------------------------------ + +Relate the Resource dispatcher to MLflow: + +.. code-block:: bash + + juju relate mlflow-server:secrets resource-dispatcher:secrets + juju relate mlflow-server:pod-defaults resource-dispatcher:pod-defaults + +Integrate MLflow with Kubeflow notebook +--------------------------------------- + +Please refer to this doc: :doc:`../tutorial/mlflow-kubeflow`. \ No newline at end of file diff --git a/docs/how-to/migrate-v1-v2.rst b/docs/how-to/migrate-v1-v2.rst new file mode 100644 index 00000000..ed0bc17a --- /dev/null +++ b/docs/how-to/migrate-v1-v2.rst @@ -0,0 +1,131 @@ +Migrate Charmed MLflow Version 1 to Version 2 +===================================================== + +This guide shows how to migrate Charmed MLflow version 1 to version 2. This guide assumes you are running the old Charmed MLflow stack version 1, which runs with MariaDB. With MLflow version 2, we only support the MySQL integration. This guide outlines how to move data from MariaDB to MySQL and how to migrate data from version 1 to version 2.1.1. Data from the object store doesn't need to be migrated. + +Prerequisites +------------- + +This guide assumes the following: + +#. You have deployed MLflow version 1 with MariaDB, MLflow server version 1.x, and MinIO. +#. You have CLI access to the machine where the Juju controller is deployed (all commands will be executed from there). + +MariaDB Backup +-------------- + +Install the ``mysqldump`` command: + +.. code-block:: bash + + sudo apt update + sudo apt install mysql-client + +Backup the MariaDB database with the following command: + +.. code-block:: bash + + mysqldump --host= --user=root --password=root --column-statistics=0 --databases database > mlflow-db.sql + +Deploy MySQL Charm +------------------- + +Deploy the MySQL charm, which is needed for MLflow v2: + +.. code-block:: bash + + juju deploy mysql-k8s --channel 8.0/beta --series jammy --trust + +.. note:: For MLflow version ``v.2.1``, we deploy the 8.0/beta version of the charm. You may deploy a more up to date version in your case. + +Please wait until the charm goes to active in ``juju status``. Then run the following command to get the password for MySQL: + +.. code-block:: bash + + juju run-action mysql-k8s/0 get-password --wait + +Adjust the Database Backup +-------------------------- + +Rename the database from ``database`` (used in MariaDB) to ``mlflow`` (used in MySQL): + +.. code-block:: bash + + sed 's/`database`/`mlflow`/g' mlflow-db.sql > mlflow-db-updated.sql + +Rename any duplicate constraints as MySQL does not allow that. In practice, the only duplicate constraint we've encountered is ``CONSTRAINT_1``. It has two occurrences. The first occurrence can be renamed to ``CONSTRAINT-1``, for example: + +.. code-block:: bash + + sed -i '0,/`CONSTRAINT_1`/s//`CONSTRAINT-1`/' mlflow-db-updated.sql + +You can do all the above modifications in the text editor of your choice if you prefer. + +Move Database to MySQL +---------------------- + +Install the MySQL CLI tool: + +.. code-block:: bash + + sudo apt update + sudo apt-get install mysql-shell + +Connect to the MySQL charm: + +.. code-block:: bash + + mysql --user=root --host= -p + # you will be prompted for password + +Create the MySQL database called ``mlflow``: + +.. code-block:: bash + + CREATE DATABASE mlflow; + +Leave the client with ``ctrl + D``. + +Move the updated database dump file to MySQL: + +.. code-block:: bash + + mysql -u root -p mlflow @/mlflow + +Update MLflow Server +--------------------- + +Remove relations from the old MLflow server: + +.. code-block:: bash + + juju remove-relation mlflow-db:mysql mlflow-server:db + juju remove-relation minio mlflow-server + +Update the MLflow server: + +.. code-block:: bash + + juju refresh mlflow-server --channel 2.1/edge + +Create relations with MinIO and MySQL: + +.. code-block:: bash + + juju relate mysql-k8s mlflow-server + juju relate minio mlflow-server diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..f35b2535 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,62 @@ +:relatedlinks: [Diátaxis](https://diataxis.fr/) + +.. _home: + +Charmed MLflow Documentation +============================ + +Charmed MLflow is a platform for managing the end-to-end machine learning lifecycle. + +It provides tools for tracking experiments, packaging code into reproducible runs, and sharing and deploying models. It integrates with popular machine learning frameworks. + +It addresses a number of common machine learning challenges: collaboration, reproducibility, maintenance, organisation and scaling. + +It's ideal for data scientists, ML engineers, hobbyists and teams looking to optimise their ML workflows with charms. + +--------- + +In this documentation +--------------------- + +.. grid:: 1 1 2 2 + + .. grid-item:: :doc:`Tutorial ` + + **Start here**: a hands-on introduction to Charmed MLflow for newcomers + + .. grid-item:: :doc:`How-to guides ` + + **Step-by-step guides** covering key operations and common tasks in Charmed MLflow + +.. grid:: 1 1 2 2 + :reverse: + + .. grid-item:: :doc:`Reference ` + + **Technical information** - specifications, APIs, architecture of Charmed MLflow + + .. grid-item:: :doc:`Explanation ` + + **Discussion and clarification** of key Charmed MLflow concepts and features + +--------- + +Project and community +--------------------- + +Charmed MLflow is an open-source project that values its community. We warmly welcome contributions, suggestions, fixes, and constructive feedback from everyone. + +* `Code of conduct`_ +* `Contribute`_ +* `Join our online chat`_ +* `Upstream Project`_ +* `Discourse Forum`_ + +.. toctree:: + :hidden: + :maxdepth: 2 + + tutorial/index + how-to/index + reference/index + explanation/index \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/reference/index.rst b/docs/reference/index.rst new file mode 100644 index 00000000..27ebe866 --- /dev/null +++ b/docs/reference/index.rst @@ -0,0 +1,4 @@ +Reference +========= + +Coming soon. \ No newline at end of file diff --git a/docs/reuse/links.txt b/docs/reuse/links.txt new file mode 100644 index 00000000..10e483ae --- /dev/null +++ b/docs/reuse/links.txt @@ -0,0 +1,10 @@ +.. _reStructuredText style guide: https://canonical-documentation-with-sphinx-and-readthedocscom.readthedocs-hosted.com/style-guide/ +.. _Read the Docs at Canonical: https://library.canonical.com/documentation/read-the-docs +.. _How to publish documentation on Read the Docs: https://library.canonical.com/documentation/publish-on-read-the-docs +.. _Example product documentation: https://canonical-example-product-documentation.readthedocs-hosted.com/ + +.. _Code of conduct: https://ubuntu.com/community/ethos/code-of-conduct +.. _Join our online chat: https://chat.charmhub.io/charmhub/channels/charmed-mlops +.. _Contribute: https://github.com/canonical/mlflow-operator/blob/main/CONTRIBUTING.md +.. _Upstream Project: https://mlflow.org/ +.. _Discourse Forum: https://discourse.charmhub.io/tag/mlflow \ No newline at end of file diff --git a/docs/tutorial/index.rst b/docs/tutorial/index.rst new file mode 100644 index 00000000..cf9a0654 --- /dev/null +++ b/docs/tutorial/index.rst @@ -0,0 +1,18 @@ +Tutorial +======== + +Step-by-step guides to help you get started with deploying and managing machine learning workflows using Charmed MLflow. + +We provide two pathways. For users who just want to try MLflow: + +.. toctree:: + :maxdepth: 1 + + mlflow + +For users who want to try MLflow and Kubeflow together: + +.. toctree:: + :maxdepth: 1 + + mlflow-kubeflow \ No newline at end of file diff --git a/docs/tutorial/mlflow-kubeflow.rst b/docs/tutorial/mlflow-kubeflow.rst new file mode 100644 index 00000000..b7c9b4c0 --- /dev/null +++ b/docs/tutorial/mlflow-kubeflow.rst @@ -0,0 +1,185 @@ +Getting Started with Charmed MLflow and Kubeflow +================================================ + ++-----------+---------+ +| Component | Version | ++-----------+---------+ +| MLflow | 2 | ++-----------+---------+ +| Kubeflow | 1.7 | ++-----------+---------+ + +Welcome to this tutorial on getting started with Charmed MLflow alongside Charmed Kubeflow. If you would like to deploy Kubeflow by itself, see our tutorial on `Charmed Kubeflow `_. + +Prerequisites +------------- + +This tutorial assumes you will be deploying Kubeflow and MLflow on a public cloud Virtual Machine (VM) with the following specs: + +- Runs Ubuntu 20.04 (focal) or later. +- Has at least 4 cores, 32GB RAM and 100GB of disk space available. +- Is connected to the internet for downloading the required snaps and charms. + +We'll also assume that you have a laptop that meets the following conditions: + +- Has an SSH tunnel open to the VM with port forwarding and a SOCKS proxy. To see how to set this up, see `How to setup SSH VM Access `_. +- Runs Ubuntu 20.04 (focal) or later. +- Has a web browser installed e.g. Chrome / Firefox / Edge. + +In the remainder of this tutorial, unless otherwise stated, it is assumed you will be running all command line operations on the VM, through the open SSH tunnel. It's also assumed you'll be using the web browser on your local machine to access the Kubeflow and MLflow dashboards. + +Deploy MLflow +------------- + +Follow the steps in this tutorial to deploy MLflow on your VM: :doc:`mlflow`. Before moving on with this tutorial, confirm that you can now access the MLflow UI on ``http://localhost:31380``. + +Deploy Kubeflow bundle +---------------------- + +Let's deploy Charmed Kubeflow alongside MLflow. Run the following command to initiate the deployment: + +.. code-block:: bash + + juju deploy kubeflow --trust --channel=1.7/stable + +Configure Dashboard Access +-------------------------- + +Run the following commands: + +.. code-block:: bash + + juju config dex-auth public-url=http://10.64.140.43.nip.io + juju config oidc-gatekeeper public-url=http://10.64.140.43.nip.io + +This tells the authentication and authorisation components of the bundle that users who access the bundle will be doing so via the URL ``http://10.64.140.43.nip.io``. In turn, this allows those components to construct appropriate responses to incoming traffic. + +Now set the dashboard username and password: + +.. code-block:: bash + + juju config dex-auth static-username=user123@email.com + juju config dex-auth static-password=user123 + +Deploy Resource Dispatcher +-------------------------- + +Next, let's deploy the resource dispatcher. The resource dispatcher is an optional component which will distribute Kubernetes objects related to MLflow credentials to all user namespaces in Kubeflow. This means that all your Kubeflow users can access the MLflow model registry from their namespaces. To deploy the dispatcher, run the following command: + +.. code-block:: bash + + juju deploy resource-dispatcher --channel 1.0/stable --trust + +This will deploy the latest edge version of the dispatcher. See `Resource Dispatcher on GitHub `_ for more info. Now we must relate the dispatcher to MLflow: + +.. code-block:: bash + + juju relate mlflow-server:secrets resource-dispatcher:secrets + juju relate mlflow-server:pod-defaults resource-dispatcher:pod-defaults + +Monitor The Deployment +---------------------- + +Now, at this point, we've deployed MLflow and Kubeflow and we've related them via the resource dispatcher. But that doesn't mean our system is ready yet: Juju will need to download charm data from CharmHub and the charms themselves will take some time to initialise. + +So, how do you know when all the charms are ready, then? You can do this using the ``juju status`` command. First, let's run a basic status command and review the output. Run the following command to print out the status of all the components of Juju: + +.. code-block:: bash + + juju status + +Review the output for yourself. You should see some summary information, a list of Apps and associated information, and another list of Units and their associated information. Don't worry too much about what this all means for now. If you're interested in learning more about this command and its output, see the `Juju Status command `_. + +The main thing we're interested in at this stage is the statuses of all the applications and units running through Juju. We want all the statuses to eventually become ``active``, indicating that the bundle is ready. Run the following command to keep a watch on the components which are not active yet: + +.. code-block:: bash + + watch -c 'juju status --color | grep -E "blocked|error|maintenance|waiting|App|Unit"' + +This will periodically run a ``juju status`` command and filter to components which are in a state of ``blocked``, ``error``, ``maintenance`` or ``waiting`` i.e. not ``active``. When this output becomes empty except for the “App” and “Unit” headings, then we know all statuses are active and our system is ready. + +Don't be surprised if some of the components' statuses change to ``blocked`` or ``error`` every now and then. This is expected behaviour, and these statuses should resolve by themselves as the bundle configures itself. However, if components remain stuck in the same error states, consult the troubleshooting steps below. + +.. dropdown:: Expand to troubleshoot: Waiting for gateway relation + + An issue you might have is the ``tensorboard-controller`` component might be stuck with a status of ``waiting`` and a message “Waiting for gateway relation”. To fix this, run: + + .. code-block:: bash + + juju run --unit istio-pilot/0 -- "export JUJU_DISPATCH_PATH=hooks/config-changed; ./dispatch" + + This is a known issue, see `TensorBoard controller GitHub issue `_ for more info. + +Be patient, it can take up to an hour for all those charms to download and initialise. In the meantime, why not try our `Juju tutorial `_? + +Integrate MLflow with Notebook +------------------------------ + +In this section, we're going to create a notebook server in Kubeflow and connect it to MLflow. This will allow our notebook logic to talk to MLflow in the background. Let's get started. + +First, to be able to use MLflow credentials in your Kubeflow notebook, visit the dashboard at ``http://10.64.140.43.nip.io/`` and fill the username and password which you configured in the previous section e.g. ``user123@email.com`` and ``user123``. + +Click on start setup to setup the Kubeflow user for the first time. + +Select ``Finish`` to finish the process. + +Now a Kubernetes namespace was created for your user. To use MLflow with this user, label the namespace with the following command: + +.. code-block:: bash + + microk8s kubectl label ns user123 user.kubeflow.org/enabled="true" + +You will get the following output: ``namespace/user123 labeled``. + +For more info on the label command, check `Kubernetes labels `_. For more info on Kubernetes namespaces for users, see the `upstream docs on Multi-user isolation `_. + +Now go back to the Dashboard. From the left panel, choose notebooks. Select +New Notebook. + +At this point, we can name the notebook as we want, and choose the desired image and resource limits. For now, let's just keep things simple: + +1. For ``Name``, enter ``test-notebook``. +2. Expand the *Custom Notebook* section and for ``image``, select ``kubeflownotebookswg/jupyter-tensorflow-full:v1.7.0``. + +Now, in order to allow our notebook server access to MLflow, we need to enable some special configuration options. Scroll down to ``Data Volumes -> Advanced options`` and from the ``Configurations`` dropdown, choose the following options: + +1. Allow access to Kubeflow pipelines. +2. Allow access to MinIO. +3. Allow access to MLflow. + +.. note:: Remember we related Kubeflow to MLflow earlier using the resource dispatcher? This is why we're seeing the MinIO and MLflow options in the dropdown! + +Great, that's all the configuration for the notebook server done. Hit the Launch button to launch the notebook server. Be patient, the notebook server will take a little while to initialise. + +When the notebook server is ready, you'll see it listed in the Notebooks table with a success status. At this point, select ``Connect`` to connect to the notebook server. + +When you connect to the notebook server, you'll be taken to the notebook environment in a new tab. Because of our earlier configurations, this environment is now connected to MLflow in the background. This means the notebooks we create here can access MLflow. Cool! + +To test this, create a new notebook and paste the following command into it, in a cell: + +.. code-block:: bash + + !printenv | grep MLFLOW + +Run the cell. This will print out two environment variables ``MLFLOW_S3_ENDPOINT_URL`` and ``MLFLOW_TRACKING_URI``, confirming MLflow is indeed connected. + +Great, we've launched a notebook server that's connected to MLflow! Now let's upload some example notebooks to this server to see MLflow in practice. + +Run MLflow examples +------------------- + +To run MLflow examples on your newly created notebook server, click on the source control icon in the leftmost navigation bar. + +From the menu, choose the ``Clone a Repository`` option. + +Now insert this repository address ``https://github.com/canonical/kubeflow-examples.git``. + +This will clone a whole ``kubeflow-examples`` repository onto the notebook server. The cloned repository will be a folder on the server, with the same name as the remote repository. Go inside the folder and after that, choose the ``mlflow-v2-examples`` sub-folder. + +There you will find two notebooks: + +- ``notebook-example.ipynb``: demonstrates how to talk to MLflow from inside a notebook. The example uses a simple classifier which is stored in the MLflow registry. +- ``pipeline-example.ipynb``: demonstrates how to talk to MLflow from a Kubeflow pipeline. The example creates and executes a three-step Kubeflow pipeline with the last step writing a model object to the MLflow registry. + +Go ahead, try those notebooks out for yourself! You can run them cell by cell using the run button, or all at once using the double chevron `>>`. + +.. note:: If you get an error in the Notebooks related to ``sklearn``, try replacing ``sklearn`` with ``scikit-learn``. See `here `_ for more details. diff --git a/docs/tutorial/mlflow.rst b/docs/tutorial/mlflow.rst new file mode 100644 index 00000000..2f48c588 --- /dev/null +++ b/docs/tutorial/mlflow.rst @@ -0,0 +1,145 @@ +Get Started with Charmed MLflow +================================== + ++-----------+---------+ +| Component | Version | ++-----------+---------+ +| MLflow | 2 | ++-----------+---------+ + +Welcome to the tutorial on Charmed MLflow! `MLflow `_ is an open-source platform, used for managing machine learning workflows. It has four primary functions that include experiment tracking, model registry, model management and code reproducibility. + +So wait, what does "Charmed MLflow" mean? Is it the same thing as MLflow? Yes and no. MLflow is a complex application, consisting of many components running together and communicating with each other. Charmed MLflow is a `charm bundle `_ that allows us to deploy MLflow quickly and easily. Don't worry too much about what a "charm bundle" is right now. The key thing is that it's going to make deploying MLflow very convenient for us: we'll get MLflow up and running with just a few command line commands! + +In this tutorial, we're going to explore Charmed MLflow in a practical way. Using the `Juju `_ CLI tool, we'll deploy MLflow to a local `MicroK8s `_ cloud. + +Prerequisites +------------- + +We are assuming that you are running this tutorial on a local machine with the following specs: + +* Runs Ubuntu 22.04 or later +* Has at least 50GB free disk space + +Install and prepare MicroK8s +---------------------------- +Let's install MicroK8s. MicroK8s is installed from a snap package. The published snap maintains different ``channels`` for different releases of Kubernetes. + +.. code-block:: bash + + sudo snap install microk8s --classic --channel=1.24/stable + +For MicroK8s to work without having to use ``sudo`` for every command, it creates a group called ``microk8s``. To make it more convenient to run commands, you will add the current user to this group: + +.. code-block:: bash + + sudo usermod -a -G microk8s $USER + newgrp microk8s + +It is also useful to make sure the user has the proper access and ownership of any ``kubectl`` configuration files: + +.. code-block:: bash + + sudo chown -f -R $USER ~/.kube + +Enable the following MicroK8s addons to configure your Kubernetes cluster with extra services needed to run Charmed Kubeflow. + +.. code-block:: bash + + microk8s enable dns hostpath-storage ingress metallb:10.64.140.43-10.64.140.49 + +Here, we added a ``dns`` service, so the applications can find each other, storage, an ingress controller so we can access Kubeflow components and the ``MetalLB`` load-balancer application. +You can see that we added some detail when enabling ``MetalLB``, in this case the address pool to use. + +> See More : `MicroK8s | How to use addons `_ + +We've now installed and configured MicroK8s. It will start running automatically, but can take 5 minutes or so before it's ready for action. Run the following command to tell MicroK8s to report its status to us when it's ready: + +.. code-block:: bash + + microk8s status --wait-ready + +Be patient - this command may not return straight away. The ``--wait-ready`` flag tells MicroK8s to wait for the Kubernetes services to initialise before returning. Once MicroK8s is ready, you will see something like the following output: + +.. code-block:: bash + + microk8s is running + +Below this there will be a bunch of other information about the cluster. + +Great, we have now installed and configured MicroK8s, and it's running and ready! + +Install Juju +------------ +`Juju `_ is an operation Lifecycle manager (OLM) for clouds, bare metal or Kubernetes. We will be using it to deploy and manage the components which make up Kubeflow. + +To install Juju from snap, run this command: + +.. code-block:: bash + + sudo snap install juju --classic --channel=2.9/stable + +Now, run the following command to deploy a Juju controller to the Kubernetes we set up with MicroK8s: + +.. code-block:: bash + + juju bootstrap microk8s + +Sit tight while the command completes! The controller may take a minute or two to deploy. + +The controller is the agent of Juju, running on Kubernetes, which can be used to deploy and control the components of Kubeflow. + +Next, we'll need to add a model for Kubeflow to the controller. Run the following command to add a model called ``kubeflow``: + +.. code-block:: bash + + juju add-model kubeflow + +.. note:: The model name here can be anything. We're just using ``kubeflow`` because often you may want to deploy MLflow along with Kubeflow, and in that case, the model name must be ``kubeflow``. So it's not a bad habit to have. + +The controller can work with different ``models``, which map 1:1 to namespaces in Kubernetes. In this case, the model name must be ``kubeflow``, due to an assumption made in the upstream Kubeflow Dashboard code. + +Great job: Juju has now been installed and configured for Kubeflow! + +Deploy MLflow bundle +-------------------- +Before deploying, run these commands: + +.. code-block:: bash + + sudo sysctl fs.inotify.max_user_instances=1280 + sudo sysctl fs.inotify.max_user_watches=655360 + +We need to run the above because under the hood, MicroK8s uses ``inotify`` to interact with the filesystem, and in large MicroK8s deployments sometimes the default ``inotify`` limits are exceeded. + +Let's now use Juju to deploy Charmed MLflow. Run the following command: + +.. code-block:: bash + + juju deploy mlflow --channel=2.1/stable --trust + +This deploys the latest edge version of MLflow with `MinIO `_ as object storage and `MySQL `_ as metadata store. + +Access MLflow +------------- +To access MLflow, visit the following URL in your web browser: + +.. code-block:: bash + + http://localhost:31380/ + +This will take you to the MLflow UI. + +.. note:: by default Charmed MLflow creates a `NodePort `_ on port 31380 where you can access the MLflow UI. + +That's it! Charmed MLflow has been deployed locally with MicroK8s and Juju. You can now start using MLflow. + +Reference: Object storage credentials +------------------------------------- +To use MLflow you need to have credentials to the object storage. The aforementioned bundle comes with MinIO. To get the ``MinIO`` credentials run the following command: + +.. code-block:: bash + + juju run-action mlflow-server/0 get-minio-credentials --wait + +This action will output ``secret-key`` and ``secret-access-key``.