From 717b989e40861ed04b13899e5583e280bebb6870 Mon Sep 17 00:00:00 2001 From: Chris White Date: Wed, 31 Oct 2018 13:51:47 -0700 Subject: [PATCH 1/2] Update dotdict doc and upgrade cloudpickle --- setup.py | 2 +- src/prefect/utilities/collections.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5f440593dd84..7ef46d130b03 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ install_requires = [ "click >= 6.7, < 7.0", - "cloudpickle == 0.5.3", + "cloudpickle >= 0.6.0", "croniter >= 0.3.23, < 0.4", "cryptography >= 2.2.2, < 3.0", "dask >= 0.18, < 0.19", diff --git a/src/prefect/utilities/collections.py b/src/prefect/utilities/collections.py index 4373ed2e722d..0c5a0b835ff0 100644 --- a/src/prefect/utilities/collections.py +++ b/src/prefect/utilities/collections.py @@ -36,7 +36,9 @@ def flatten_seq(seq: Iterable) -> Generator: class DotDict(MutableMapping): """ A `dict` that also supports attribute ("dot") access. Think of this as an extension - to the standard python `dict` object. + to the standard python `dict` object. **Note**: while any hashable object can be added to + a `DotDict`, _only_ valid Python identifiers can be accessed with the dot syntax; this excludes + strings which begin in numbers, special characters, or double underscores. Args: - init_dict (dict, optional): dictionary to initialize the `DotDict` From c50c53aaebebf91fc3021f4c5d93fe9de97a46ec Mon Sep 17 00:00:00 2001 From: Chris White Date: Wed, 31 Oct 2018 15:55:25 -0700 Subject: [PATCH 2/2] Update CHANGELOG with every doc build --- .gitignore | 1 + docs/changelog.md | 121 ------------------------------------------ docs/generate_docs.py | 19 +++++++ 3 files changed, 20 insertions(+), 121 deletions(-) delete mode 100644 docs/changelog.md diff --git a/.gitignore b/.gitignore index 42d92ca5c375..6f46e2322770 100644 --- a/.gitignore +++ b/.gitignore @@ -120,5 +120,6 @@ node_modules # docs docs/api +docs/changelog.md docs/deploy/dist docs/.vuepress/public/prefect-coverage diff --git a/docs/changelog.md b/docs/changelog.md deleted file mode 100644 index 0f7d5473eef4..000000000000 --- a/docs/changelog.md +++ /dev/null @@ -1,121 +0,0 @@ -# Prefect Changelog - -## 0.3.3 - -### Major Features - -- Refactor `FlowRunner` and `TaskRunner` into a modular `Runner` pipelines - [#260](https://github.com/PrefectHQ/prefect/pull/260), [#267](https://github.com/PrefectHQ/prefect/pull/267) -- Add configurable `state_handlers` for `FlowRunners`, `Flows`, `TaskRunners`, and `Tasks` - [#264](https://github.com/PrefectHQ/prefect/pull/264), [#267](https://github.com/PrefectHQ/prefect/pull/267) -- Add gmail and slack notification state handlers w/ tutorial - [#274](https://github.com/PrefectHQ/prefect/pull/274), [#294](https://github.com/PrefectHQ/prefect/pull/294) - -### Minor Features - -- Add a new method `flow.get_tasks()` for easily filtering flow tasks by attribute - [#242](https://github.com/PrefectHQ/prefect/pull/242) -- Add new `JinjaTemplateTask` for easily rendering jinja templates - [#200](https://github.com/PrefectHQ/prefect/issues/200) -- Add new `PAUSE` signal for halting task execution - [#246](https://github.com/PrefectHQ/prefect/pull/246) -- Add new `Paused` state corresponding to `PAUSE` signal, and new `pause_task` utility - [#251](https://github.com/PrefectHQ/prefect/issues/251) -- Add ability to timeout task execution for all executors except `DaskExecutor(processes=True)` - [#240](https://github.com/PrefectHQ/prefect/issues/240) -- Add explicit unit test to check Black formatting (Python 3.6+) - [#261](https://github.com/PrefectHQ/prefect/pull/261) -- Add ability to set local secrets in user config file - [#231](https://github.com/PrefectHQ/prefect/issues/231), [#274](https://github.com/PrefectHQ/prefect/pull/274) -- Add `is_skipped()` and `is_scheduled()` methods for `State` objects - [#266](https://github.com/PrefectHQ/prefect/pull/266), [#278](https://github.com/PrefectHQ/prefect/pull/278) -- Adds `now()` as a default `start_time` for `Scheduled` states - [#278](https://github.com/PrefectHQ/prefect/pull/278) -- `Signal` classes now pass arguments to underlying `State` objects - [#279](https://github.com/PrefectHQ/prefect/pull/279) -- Run counts are tracked via `Retrying` states - [#281](https://github.com/PrefectHQ/prefect/pull/281) - -### Fixes - -- Flow consistently raises if passed a parameter that doesn't exist - [#149](https://github.com/PrefectHQ/prefect/issues/149) - -### Breaking Changes - -- Renamed `scheduled_time` -> `start_time` in `Scheduled` state objects - [#278](https://github.com/PrefectHQ/prefect/pull/278) -- `TaskRunner.check_for_retry` no longer checks for `Retry` states without `start_time` set - [#278](https://github.com/PrefectHQ/prefect/pull/278) -- Swapped the position of `result` and `message` attributes in State initializations, and started storing caught exceptions as results - [#283](https://github.com/PrefectHQ/prefect/issues/283) - -## 0.3.2 - -### Major Features - -- Local parallelism with `DaskExecutor` - [#151](https://github.com/PrefectHQ/prefect/issues/151), [#186](https://github.com/PrefectHQ/prefect/issues/186) -- Resource throttling based on `tags` - [#158](https://github.com/PrefectHQ/prefect/issues/158), [#186](https://github.com/PrefectHQ/prefect/issues/186) -- `Task.map` for mapping tasks - [#186](https://github.com/PrefectHQ/prefect/issues/186) -- Added `AirFlow` utility for importing Airflow DAGs as Prefect Flows - [#232](https://github.com/PrefectHQ/prefect/pull/232) - -### Minor Features - -- Use Netlify to deploy docs - [#156](https://github.com/prefecthq/prefect/issues/156) -- Add changelog - [#153](https://github.com/prefecthq/prefect/issues/153) -- Add `ShellTask` - [#150](https://github.com/prefecthq/prefect/issues/150) -- Base `Task` class can now be run as a dummy task - [#191](https://github.com/PrefectHQ/prefect/pull/191) -- New `return_failed` keyword to `flow.run()` for returning failed tasks - [#205](https://github.com/PrefectHQ/prefect/pull/205) -- some minor changes to `flow.visualize()` for visualizing mapped tasks and coloring nodes by state - [#202](https://github.com/PrefectHQ/prefect/issues/202) -- Added new `flow.replace()` method for swapping out tasks within flows - [#230](https://github.com/PrefectHQ/prefect/pull/230) -- Add `debug` kwarg to `DaskExecutor` for optionally silencing dask logs - [#209](https://github.com/PrefectHQ/prefect/issues/209) -- Update `BokehRunner` for visualizing mapped tasks - [#220](https://github.com/PrefectHQ/prefect/issues/220) -- Env var configuration settings are typed - [#204](https://github.com/PrefectHQ/prefect/pull/204) -- Implement `map` functionality for the `LocalExecutor` - [#233](https://github.com/PrefectHQ/prefect/issues/233) - -### Fixes - -- Fix issue with Versioneer not picking up git tags - [#146](https://github.com/prefecthq/prefect/issues/146) -- `DotDicts` can have non-string keys - [#193](https://github.com/prefecthq/prefect/issues/193) -- Fix unexpected behavior in assigning tags using contextmanagers - [#190](https://github.com/PrefectHQ/prefect/issues/190) -- Fix bug in initialization of Flows with only `edges` - [#225](https://github.com/PrefectHQ/prefect/pull/225) -- Remove "bottleneck" when creating pipelines of mapped tasks - [#224](https://github.com/PrefectHQ/prefect/pull/224) - -### Breaking Changes - -- Runner refactor - [#221](https://github.com/PrefectHQ/prefect/pull/221) -- Cleaned up signatures of `TaskRunner` methods - [#171](https://github.com/prefecthq/prefect/issues/171) -- Locally, Python 3.4 users can not run the more advanced parallel executors (`DaskExecutor`) [#186](https://github.com/PrefectHQ/prefect/issues/186) - -## 0.3.1 - -### Major Features - -- Support for user configuration files - [#195](https://github.com/PrefectHQ/prefect/pull/195) - -### Minor Features - -- None - -### Fixes - -- Let DotDicts accept non-string keys - [#194](https://github.com/PrefectHQ/prefect/pull/194) - -### Breaking Changes - -- None - -## 0.3.0 - -### Major Features - -- BokehRunner - [#104](https://github.com/prefecthq/prefect/issues/104), [#128](https://github.com/prefecthq/prefect/issues/128) -- Control flow: `ifelse`, `switch`, and `merge` - [#92](https://github.com/prefecthq/prefect/issues/92) -- Set state from `reference_tasks` - [#95](https://github.com/prefecthq/prefect/issues/95), [#137](https://github.com/prefecthq/prefect/issues/137) -- Add flow `Registry` - [#90](https://github.com/prefecthq/prefect/issues/90) -- Output caching with various `cache_validators` - [#84](https://github.com/prefecthq/prefect/issues/84), [#107](https://github.com/prefecthq/prefect/issues/107) -- Dask executor - [#82](https://github.com/prefecthq/prefect/issues/82), [#86](https://github.com/prefecthq/prefect/issues/86) -- Automatic input caching for retries, manual-only triggers - [#78](https://github.com/prefecthq/prefect/issues/78) -- Functional API for `Flow` definition -- `State` classes -- `Signals` to transmit `State` - -### Minor Features - -- Add custom syntax highlighting to docs - [#141](https://github.com/prefecthq/prefect/issues/141) -- Add `bind()` method for tasks to call without copying - [#132](https://github.com/prefecthq/prefect/issues/132) -- Cache expensive flow graph methods - [#125](https://github.com/prefecthq/prefect/issues/125) -- Docker environments - [#71](https://github.com/prefecthq/prefect/issues/71) -- Automatic versioning via Versioneer - [#70](https://github.com/prefecthq/prefect/issues/70) -- `TriggerFail` state - [#67](https://github.com/prefecthq/prefect/issues/67) -- State classes - [#59](https://github.com/prefecthq/prefect/issues/59) - -### Fixes - -- None - -### Breaking Changes - -- None diff --git a/docs/generate_docs.py b/docs/generate_docs.py index 1c05b21fdcd4..2f809609624b 100644 --- a/docs/generate_docs.py +++ b/docs/generate_docs.py @@ -488,6 +488,8 @@ def get_class_methods(obj): shutil.rmtree("api", ignore_errors=True) os.makedirs("api", exist_ok=True) generate_coverage() + + ## UPDATE README with open("api/README.md", "w+") as f: f.write( textwrap.dedent( @@ -516,6 +518,23 @@ def get_class_methods(obj): f.write("\n" + readme[readme.index("# Prefect") :]) f.write(auto_generated_footer) + ## UPDATE CHANGELOG + with open("changelog.md", "w+") as f: + f.write( + textwrap.dedent( + """ + --- + sidebarDepth: 1 + editLink: false + --- + """ + ).lstrip() + ) + with open("../CHANGELOG.md", "r") as g: + changelog = g.read() + f.write(changelog) + f.write(auto_generated_footer) + for page in OUTLINE: # collect what to document fname, classes, fns = (