diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ce92b34a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.pytest_cache +.tox diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..c5885097 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,21 @@ +name: Lint + +on: push + +jobs: + formatting: + if: "!contains(github.event.head_commit.message, 'skip_ci')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install --upgrade tox + - run: tox -e checkformatting + flake8: + if: "!contains(github.event.head_commit.message, 'skip_ci')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install --upgrade tox + - run: tox -e flake8 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..d607c519 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Pytest + +on: push + +jobs: + test: + if: "!contains(github.event.head_commit.message, 'skip_ci')" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.7, 3.8] + + name: Python ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - run: python -m pip install --upgrade pip tox + - run: tox -e pytest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3fc8aaf0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.8.3 + +WORKDIR /build/julia_installer + +RUN wget -q https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.3-linux-x86_64.tar.gz &&\ + tar -xf julia-1.5.3-linux-x86_64.tar.gz -C /usr/share + +WORKDIR /build/gurobi_installer + +RUN wget -q https://packages.gurobi.com/9.1/gurobi9.1.0_linux64.tar.gz && \ + tar -xf gurobi9.1.0_linux64.tar.gz -C /usr/share + +ENV PATH="$PATH:/usr/share/julia-1.5.3/bin" \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/share/gurobi910/linux64/lib \ + GUROBI_HOME='/usr/share/gurobi910/linux64' \ + GRB_LICENSE_FILE='/usr/share/gurobi_license/gurobi.lic' \ + JULIA_PROJECT='/app' \ + PYTHONPATH=/app/pyreisejl:${PYTHONPATH} \ + FLASK_APP=pyreisejl/utility/app.py + +WORKDIR /app +COPY . . + +RUN julia -e 'using Pkg; Pkg.activate("."); Pkg.instantiate(); Pkg.add("Gurobi"); import Gurobi; using REISE' && \ + pip install -r requirements.txt + + +CMD ["flask", "run", "--host", "0.0.0.0"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..e1fda766 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Breakthrough Energy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Manifest.toml b/Manifest.toml index dfeff05b..3c5080c5 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,5 +1,11 @@ # This file is machine-generated - editing it directly is not advised +[[Artifacts]] +deps = ["Pkg"] +git-tree-sha1 = "c30985d8821e0cd73870b17b0ed0ce6dc44cb744" +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.3.0" + [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -9,23 +15,17 @@ git-tree-sha1 = "9e62e66db34540a0c919d72172cc2f642ac71260" uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" version = "0.5.0" -[[BinDeps]] -deps = ["Libdl", "Pkg", "SHA", "URIParser", "Unicode"] -git-tree-sha1 = "46cf2c1668ad07aba5a9d331bdeea994a1f13856" -uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee" -version = "1.0.1" - -[[BinaryProvider]] -deps = ["Libdl", "Logging", "SHA"] -git-tree-sha1 = "428e9106b1ff27593cbd979afac9b45b82372b8c" -uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" -version = "0.5.9" - [[Blosc]] -deps = ["BinaryProvider", "CMakeWrapper", "Libdl"] -git-tree-sha1 = "9981f1795919b8f770dc064fe733ba09c2e7c7a9" +deps = ["Blosc_jll"] +git-tree-sha1 = "84cf7d0f8fd46ca6f1b3e0305b4b4a37afe50fd6" uuid = "a74b3585-a348-5f62-a45c-50e91977d574" -version = "0.6.0" +version = "0.7.0" + +[[Blosc_jll]] +deps = ["Libdl", "Lz4_jll", "Pkg", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "aa9ef39b54a168c3df1b2911e7797e4feee50fbe" +uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" +version = "1.14.3+1" [[BufferedStreams]] deps = ["Compat", "Test"] @@ -34,28 +34,16 @@ uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" version = "1.0.0" [[Bzip2_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "3663bfffede2ef41358b6fc2e1d8a6d50b3c3904" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e" uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" -version = "1.0.6+2" - -[[CMake]] -deps = ["BinDeps"] -git-tree-sha1 = "50a8b41d2c562fccd9ab841085fc7d1e2706da82" -uuid = "631607c0-34d2-5d66-819e-eb0f9aa2061a" -version = "1.2.0" - -[[CMakeWrapper]] -deps = ["BinDeps", "CMake", "Libdl", "Parameters", "Test"] -git-tree-sha1 = "16d4acb3d37dc05b714977ffefa8890843dc8985" -uuid = "d5fb7624-851a-54ee-a528-d3f3bac0b4a0" -version = "0.2.3" +version = "1.0.6+5" [[CSV]] -deps = ["CategoricalArrays", "DataFrames", "Dates", "FilePathsBase", "Mmap", "Parsers", "PooledArrays", "Tables", "Unicode", "WeakRefStrings"] -git-tree-sha1 = "fe9b828d5e7b55431d75d6d180ef843d69dea048" +deps = ["CategoricalArrays", "DataFrames", "Dates", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode"] +git-tree-sha1 = "f095e44feec53d0ae809714a78c25908d1f370e6" uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "0.5.23" +version = "0.7.4" [[Calculus]] deps = ["LinearAlgebra"] @@ -64,63 +52,63 @@ uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" version = "0.5.1" [[CategoricalArrays]] -deps = ["Compat", "DataAPI", "Future", "JSON", "Missings", "Printf", "Reexport", "Statistics", "Unicode"] -git-tree-sha1 = "23d7324164c89638c18f6d7f90d972fa9c4fa9fb" +deps = ["DataAPI", "Future", "JSON", "Missings", "Printf", "Statistics", "StructTypes", "Unicode"] +git-tree-sha1 = "2ac27f59196a68070e132b25713f9a5bbc5fa0d2" uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" -version = "0.7.7" +version = "0.8.3" [[CodecBzip2]] deps = ["Bzip2_jll", "Libdl", "TranscodingStreams"] -git-tree-sha1 = "2fee975d68f9a8b22187ae86e33c0829b30cf231" +git-tree-sha1 = "2e62a725210ce3c3c2e1a3080190e7ca491f18d7" uuid = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" -version = "0.7.1" +version = "0.7.2" [[CodecZlib]] -deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] -git-tree-sha1 = "05916673a2627dd91b4969ff8ba6941bc85a960e" +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.6.0" +version = "0.7.0" [[CommonSubexpressions]] -deps = ["Test"] -git-tree-sha1 = "efdaf19ab11c7889334ca247ff4c9f7c322817b0" +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" -version = "0.2.0" +version = "0.3.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "fecfed095803b86cc06fd7ee09d3d2c98fad4dac" +git-tree-sha1 = "a706ff10f1cd8dab94f59fd09c0e657db8e77ff0" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.9.0" +version = "3.23.0" [[CompilerSupportLibraries_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "7c4f882c41faa72118841185afc58a2eb00ef612" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8e695f735fca77e9708e795eda62afdb869cbb70" uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "0.3.3+0" +version = "0.3.4+0" [[Conda]] deps = ["JSON", "VersionParsing"] -git-tree-sha1 = "7a58bb32ce5d85f8bf7559aa7c2842f9aecf52fc" +git-tree-sha1 = "c0647249d785f1d5139c0cc96db8f6b32f7ec416" uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" -version = "1.4.1" +version = "1.5.0" [[DataAPI]] -git-tree-sha1 = "176e23402d80e7743fc26c19c681bfb11246af32" +git-tree-sha1 = "ad84f52c0b8f05aa20839484dbaf01690b41ff84" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.3.0" +version = "1.4.0" [[DataFrames]] deps = ["CategoricalArrays", "Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "Missings", "PooledArrays", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "00136fcd39d503e66ab1b2eab800c47deaf7ca04" +git-tree-sha1 = "d4436b646615928b634b37e99a3288588072f851" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "0.20.0" +version = "0.21.4" [[DataStructures]] deps = ["InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "6166ecfaf2b8bbf2b68d791bc1d54501f345d314" +git-tree-sha1 = "88d48e133e6d3dd68183309877eac74393daa7eb" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.17.15" +version = "0.17.20" [[DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" @@ -137,53 +125,47 @@ uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" [[DiffResults]] deps = ["StaticArrays"] -git-tree-sha1 = "da24935df8e0c6cf28de340b958f6aac88eaa0cc" +git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" -version = "1.0.2" +version = "1.0.3" [[DiffRules]] deps = ["NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "eb0c34204c8410888844ada5359ac8b96292cfd1" +git-tree-sha1 = "214c3fcac57755cfda163d91c58893a8723f93e9" uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.0.1" +version = "1.0.2" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -[[FilePathsBase]] -deps = ["Dates", "LinearAlgebra", "Printf", "Test", "UUIDs"] -git-tree-sha1 = "2cd6e2e7965934f72cb80251f760228e2264bab3" -uuid = "48062228-2e41-5def-b9a4-89aafe57970f" -version = "0.7.0" - [[ForwardDiff]] deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"] -git-tree-sha1 = "869540e4367122fbffaace383a5bdc34d6e5e5ac" +git-tree-sha1 = "8de2519a83c6c1c2442c2f481dd9a8364855daf4" uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.10" +version = "0.10.14" [[Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" -[[Gurobi]] -deps = ["Libdl", "LinearAlgebra", "MathOptInterface", "MathProgBase", "SparseArrays"] -git-tree-sha1 = "3d031829eee9a2baf817065944be81bb37615201" -uuid = "2e9cd046-0924-5485-92f1-d5272153d98b" -version = "0.7.5" - [[HDF5]] -deps = ["BinaryProvider", "Blosc", "CMakeWrapper", "Libdl", "Mmap"] -git-tree-sha1 = "d3ea5532668bf9bdd5e8d5f16571e0b520cbbb9f" +deps = ["Blosc", "HDF5_jll", "Libdl", "Mmap", "Random"] +git-tree-sha1 = "0b812e7872e2199a5a04944f486b4048944f1ed8" uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -version = "0.12.5" +version = "0.13.7" + +[[HDF5_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "bc9c3d43ffd4d8988bfa372b86d4bdbd26860e95" +uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" +version = "1.10.5+7" [[HTTP]] -deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"] -git-tree-sha1 = "fe31f4ff144392ad8176f5c7c03cca6ba320271c" +deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets", "URIs"] +git-tree-sha1 = "9634200f8e16554cb1620dfb20501483b873df86" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "0.8.14" +version = "0.9.0" [[IniFile]] deps = ["Test"] @@ -206,25 +188,31 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" +[[JLLWrappers]] +git-tree-sha1 = "c70593677bbf2c3ccab4f7500d0f4dacfff7b75c" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.1.3" + [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e" +git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.0" +version = "0.21.1" [[JSONSchema]] -deps = ["BinaryProvider", "HTTP", "JSON"] -git-tree-sha1 = "b0a7f9328967df5213691d318a03cf70ea8c76b1" +deps = ["HTTP", "JSON", "ZipFile"] +git-tree-sha1 = "b84ab8139afde82c7c65ba2b792fe12e01dd7307" uuid = "7d188eb4-7ad8-530c-ae41-71a32a6d4692" -version = "0.2.0" +version = "0.3.3" [[JuMP]] -deps = ["Calculus", "DataStructures", "ForwardDiff", "LinearAlgebra", "MathOptInterface", "NaNMath", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "ba7f96010ed290d77d5c941c32e5df107ca688a4" +deps = ["Calculus", "DataStructures", "ForwardDiff", "LinearAlgebra", "MathOptInterface", "MutableArithmetics", "NaNMath", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "cbab42e2e912109d27046aa88f02a283a9abac7c" uuid = "4076af6c-e467-56ae-b986-b466b2749572" -version = "0.20.1" +version = "0.21.3" [[LibGit2]] +deps = ["Printf"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[Libdl]] @@ -237,17 +225,23 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +[[Lz4_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "51b1db0732bbdcfabb60e36095cc3ed9c0016932" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.9.2+2" + [[MAT]] deps = ["BufferedStreams", "CodecZlib", "HDF5", "SparseArrays"] -git-tree-sha1 = "33db0ab3000dabd036867b3dd09b41c29a31ed9a" +git-tree-sha1 = "7e36f6a52274ddb8515ec1f559306be3f412d6a6" uuid = "23992714-dd62-5051-b70f-ba57cb901cac" -version = "0.7.0" +version = "0.8.1" [[MacroTools]] deps = ["Markdown", "Random"] -git-tree-sha1 = "f7d2e3f654af75f01ec49be82c231c382214223a" +git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.5" +version = "0.5.6" [[Markdown]] deps = ["Base64"] @@ -255,74 +249,61 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[MathOptInterface]] deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "JSON", "JSONSchema", "LinearAlgebra", "MutableArithmetics", "OrderedCollections", "SparseArrays", "Test", "Unicode"] -git-tree-sha1 = "27f2ef85879b8f1d144266ab44f076ba0dfbd8a1" +git-tree-sha1 = "c4788b9cb29f8d1508e16419e66a7e617b46192d" uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" -version = "0.9.13" - -[[MathProgBase]] -deps = ["LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "9abbe463a1e9fc507f12a69e7f29346c2cdc472c" -uuid = "fdba3010-5040-5b88-9595-932c9decdf73" -version = "0.7.8" +version = "0.9.19" [[MbedTLS]] deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"] -git-tree-sha1 = "426a6978b03a97ceb7ead77775a1da066343ec6e" +git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe" uuid = "739be429-bea8-5141-9913-cc70e7f3736d" -version = "1.0.2" +version = "1.0.3" [[MbedTLS_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "c83f5a1d038f034ad0549f9ee4d5fac3fb429e33" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "0eef589dd1c26a3ac9d753fe1a8bcad63f956fa6" uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.16.0+2" +version = "2.16.8+1" [[Missings]] deps = ["DataAPI"] -git-tree-sha1 = "de0a5ce9e5289f27df672ffabef4d1e5861247d5" +git-tree-sha1 = "ed61674a0864832495ffe0a7e889c0da76b0f4c8" uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "0.4.3" +version = "0.4.4" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[MutableArithmetics]] deps = ["LinearAlgebra", "SparseArrays", "Test"] -git-tree-sha1 = "e1edd618a8f39d16f8595dd622a63b25f759cf8a" +git-tree-sha1 = "7631203bddc2424717fa42ecda2a5fed9ff36af0" uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" -version = "0.2.9" +version = "0.2.12" [[NaNMath]] -git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f" +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "0.3.3" +version = "0.3.5" [[OpenSpecFun_jll]] -deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"] -git-tree-sha1 = "d51c416559217d974a1113522d5919235ae67a87" +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9db77584158d0ab52307f8c04f8e7c08ca76b5b3" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" -version = "0.5.3+3" +version = "0.5.3+4" [[OrderedCollections]] -deps = ["Random", "Serialization", "Test"] -git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" +git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.1.0" - -[[Parameters]] -deps = ["OrderedCollections"] -git-tree-sha1 = "b62b2558efb1eef1fa44e4be5ff58a515c287e38" -uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" -version = "0.12.0" +version = "1.3.2" [[Parsers]] -deps = ["Dates", "Test"] -git-tree-sha1 = "0c16b3179190d3046c073440d94172cfc3bb0553" +deps = ["Dates"] +git-tree-sha1 = "b417be52e8be24e916e34b3d70ec2da7bdf56a68" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "0.3.12" +version = "1.0.12" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Test", "UUIDs"] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[PooledArrays]] @@ -355,9 +336,21 @@ git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "0.2.0" +[[Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "cfbac6c1ed70c002ec6361e7fd334f02820d6419" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.1.2" + [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +[[SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "6ccde405cf0759eba835eb613130723cb8f10ff9" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.2.16" + [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -380,20 +373,26 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[SpecialFunctions]] deps = ["OpenSpecFun_jll"] -git-tree-sha1 = "e19b98acb182567bcb7b75bb5d9eedf3a3b5ec6c" +git-tree-sha1 = "7286f31f27e3335cba31c618ac344a35eceac060" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "0.10.0" +version = "1.1.0" [[StaticArrays]] deps = ["LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "4118cba3529e99af61aea9a83f7bfd3cff5ffb28" +git-tree-sha1 = "794e88df12426adc7eb3370b4ec3aee12f8d6911" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "0.12.2" +version = "1.0.0" [[Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[[StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "1ed04f622a39d2e5a6747c3a70be040c00333933" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.1.0" + [[TableTraits]] deps = ["IteratorInterfaceExtensions"] git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e" @@ -402,9 +401,9 @@ version = "1.0.0" [[Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"] -git-tree-sha1 = "aaed7b3b00248ff6a794375ad6adf30f30ca5591" +git-tree-sha1 = "5131a624173d532299d1c7eb05341c18112b21b8" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "0.2.11" +version = "1.2.1" [[Test]] deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] @@ -416,11 +415,10 @@ git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.9.5" -[[URIParser]] -deps = ["Unicode"] -git-tree-sha1 = "53a9f49546b8d2dd2e688d216421d050c9a31d0d" -uuid = "30578b45-9adc-5946-b283-645ec420af67" -version = "0.4.1" +[[URIs]] +git-tree-sha1 = "bc331715463c41d601cf8bfd38ca70a490af5c5b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.1.0" [[UUIDs]] deps = ["Random", "SHA"] @@ -434,8 +432,20 @@ git-tree-sha1 = "80229be1f670524750d905f8fc8148e5a8c4537f" uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" version = "1.2.0" -[[WeakRefStrings]] -deps = ["DataAPI", "Random", "Test"] -git-tree-sha1 = "28807f85197eaad3cbd2330386fac1dcb9e7e11d" -uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" -version = "0.6.2" +[[ZipFile]] +deps = ["Libdl", "Printf", "Zlib_jll"] +git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7" +uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" +version = "0.9.3" + +[[Zlib_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "320228915c8debb12cb434c59057290f0834dbf6" +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.11+18" + +[[Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6f1abcb0c44f184690912aa4b0ba861dd64f11b9" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.4.5+2" diff --git a/Project.toml b/Project.toml index f99e145c..4fad8e16 100644 --- a/Project.toml +++ b/Project.toml @@ -1,19 +1,19 @@ name = "REISE" uuid = "b654829c-3c70-409d-9aea-4c9c04976dc6" -authors = ["Daniel Olsen "] -version = "0.1.0" +authors = ["Daniel Olsen "] +version = "0.2.0" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" MAT = "23992714-dd62-5051-b70f-ba57cb901cac" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -DataFrames = "0.20" -JuMP = "0.20" +DataFrames = "0.21" +JuMP = "0.21.3" diff --git a/README.md b/README.md index ace88594..c68ebc3f 100644 --- a/README.md +++ b/README.md @@ -1,108 +1,527 @@ -## Installation +# REISE.jl +Renewable Energy Integration Simulation Engine. -### Julia package +This repository contains, in the **src** folder, the Julia scripts to run the power-flow study in the U.S. electric grid. The simulation engine relies on [Gurobi] as the optimization solver. -The most reliable way to install this package is by cloning the repo locally, -navigating to the project folder, activating the project, and instantiating it. -This approach will copy install all dependencies in the **exact** version as -they were installed during package development. **Note**: If `Gurobi.jl` is not -already installed in your Julia environment, then its build step will fail if -it cannot find the Gurobi installation folder. To avoid this, you can specify -an environment variable for `GUROBI_HOME`, pointing to the Gurobi -``. -For more information, see https://github.com/JuliaOpt/Gurobi.jl#installation. -To instantiate: +## Table of Contents +1. [Dependencies](#dependencies) +2. [Installation (Local)](#installation-local) +3. [Usage (Julia)](#usage-julia) +4. [Usage (Python)](#usage-python) +5. [Docker](#docker) +6. [Package Structure](#package-structure) +7. [Formulation](#formulation) + +## Dependencies +This package requires installations of the following, with recommended versions listed. +- [Julia], version 1.5 +- [Python], version 3.8 + +An external solver is required to run optimizations. We recommend [Gurobi] (version 9.1), though any other solver that is [compatible with JuMP] can be used. +Note: as Gurobi is a commercial solver, a [Gurobi license file] is required. This may be either a local license or a [Gurobi Cloud license]. [Free licenses for academic use] are available. + +This package can also be run using [Docker], which will automatically handle the installation of Julia, Python, and all dependencies. Gurobi is also installed, although as before a [Gurobi license file] is still required to use Gurobi as a solver; other solvers can also be used. + +For sample data to use with the simulation, please visit [Zenodo]. + +### System Requirements + +Large simulations can require significant amounts of RAM. The amount of RAM necessary is proportional to both the size of the grid and the size of the interval with which to run the simulation. + +As a general estimate, 1-2 GB of RAM is needed per hour in the interval in a simulation across the entire USA grid. For example, a 24-hour interval would require 24-48 GB of RAM; if only 16 GB of RAM is available, consider using a time interval of 8 hours or less as that would take 8-16 GB of RAM. + +The memory necessary would also be proportional to the size of grid used, so as the Western interconnect is roughly 8 times smaller than the entire USA grid, a simulation of just the Western interconnect with a 24-hour interval would require ~3-6 GB of RAM. + +## Installation (Local) + +When installing this package locally, the below dependencies will need to be installed following the provider recommendations: +- [Download Julia] +- [Download Python] + +If Gurobi is to be used as the solver, this will need to be installed as well: +- [Gurobi Installation Guide] + +The package itself has two components that require installation: +- [`Julia` package](#julia-package-installation) to run the simulation +- optional [`python` scripts](#python-requirements-installation) for some additional pre- and post-processing + +Instead of installing locally, this package can also be used with the included [Docker](#Docker) image. + +Detailed installation instructions for both the necessary applications and packages can be found below: +1. [Gurobi Installation](#gurobi-installation) + 1. [Gurobi Installation Example (Linux + Cloud License)](#gurobi-installation-example-linux-cloud-license) + 2. [Gurobi Installation Verification](#gurobi-verification) +2. [Julia Installation](#julia-installation) + 1. [Julia Installation Example (Linux)](#julia-installation-example-linux) + 2. [Julia Package Installation](#julia-package-installation) + 3. [Julia Installation Verification](#julia-verification) +3. [Python Installation](#python-installation) + 1. [Python Requirements Installation](#python-requirements-installation) + 2. [Python Installation Verification](#python-installation-verification) + +### Gurobi Installation +Installation of `Gurobi` depends on both your operating system and license type. Detailed instructions can be found at the [Gurobi Installation Guide]. + +#### Gurobi Installation Example (Linux + Cloud License) + +1. Choose a destination directory for `Gurobi`. For a shared installation, `/opt` is recommended. +```bash +cd /opt ``` -pkg> activate . -(REISE) pkg> instantiate +2. Download and unzip the `Gurobi` package in the chosen directory. +```bash +wget https://packages.gurobi.com/9.1/gurobi9.1.0_linux64.tar.gz +tar -xvfz gurobi9.1.0_linux64.tar.gz +``` +This will create a subdirectory, `/opt/gurobi910/linux64` in which the complete distribution is located. This will be considered the `` in the rest of this section. + +2. Set environmental variables for `Gurobi`: +- `GUROBI_HOME` should be set to your ``. +- `PATH` should be extended to include `/bin`.` +- `LD_LIBRARY_PATH` should be extended to include /lib. + +For bash shell users, add the following to the `.bashrc` file: +```bash +export GUROBI_HOME="/opt/gurobi910/linux64" +export PATH="${PATH}:${GUROBI_HOME}/bin" +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${GUROBI_HOME}/lib" ``` -Another way is to install the package using the list of dependencies specified -in the `Project.toml` file, which will pull the most recent allowed version of -the dependencies. Currently, this package is known to be compatible with JuMP -v0.20, but not v0.21; this is specified in the `Project.toml` file, but there -may be other packages for which the latest version does not maintain -backward-compatibility. +3. The `Gurobi` license needs to be download and installed. Download a copy of your Gurobi license from the account portal, and copy it to the parent directory of the ``. -This package is not registered. Therefore, it must be added to a Julia -environment either directly from github: +```bash +cp gurobi.lic /opt/gurobi910/gurobi.lic ``` -pkg> add https://github.com/intvenlab/REISE.jl#develop + +#### Gurobi Verification + +To verify that Gurobi has installed properly, run `gurobi.sh` located in the `bin` folder of the Gurobi installation. +```bash +/usr/share/gurobi910/linux64/bin/gurobi.sh ``` -or by cloning the repository locally and then specifying the path to the repo: + +An example of the expected output for this program (using a cloud license): ``` -pkg> add /YOUR_PATH_HERE/REISE.jl#develop +This program should give the following output +Python 3.7.4 (default, Oct 29 2019, 10:15:53) +[GCC 4.4.7 20120313 (Red Hat 4.4.7-18)] on linux +Type "help", "copyright", "credits" or "license" for more information. +Using license file /usr/share/gurobi_license/gurobi.lic +Set parameter CloudAccessID +Set parameter CloudSecretKey +Set parameter LogFile to value gurobi.log +Waiting for cloud server to start (pool default)... +Starting... +Starting... +Starting... +Starting... +Compute Server job ID: 1eacfb69-3083-44e2-872e-58515b143b5d +Capacity available on 'https://ip-10-0-55-163:61000' - connecting... +Established HTTPS encrypted connection + +Gurobi Interactive Shell (linux64), Version 9.1.0 +Copyright (c) 2020, Gurobi Optimization, LLC +Type "help()" for help + +gurobi> ``` -Instead of calling `add PACKAGE`, it is also possible to call `dev PACKAGE`, -which will always import the latest version of the code on your local machine. -See the documentation for the Julia package manager for more information: -https://julialang.github.io/Pkg.jl/v1/. +### Julia Installation -### Associated python scripts +[Download Julia] and install the version specific to your operating system. -The dependencies of the python scripts contained in `pyreisejl/` are not -automatically installed. See `requirements.txt` for details. +#### Julia Installation Example (Linux) -### Other tools +1. Choose a destination directory for `Julia`. Again, `/opt` is recommended. +```bash +cd /opt +``` -Text file manipulation requires GNU `awk`, also known as `gawk`. +2. Download and unzip the `Julia` package in the chosen directory. +```bash +wget -q https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.3-linux-x86_64.tar.gz +tar -xf julia-1.5.3-linux-x86_64.tar.gz +``` -## Usage +3. Add `Julia` to the `PATH` environmental variable. -Installation registers a package named `REISE`. Following Julia naming -conventions, the `.jl` is dropped. The package can be imported using: -`import REISE` to call `REISE.run_scenario()`, or `using REISE` to call -`run_scenario()`. +For bash shell users, add the following to the `.bashrc` file: +```bash +export PATH="$PATH:/opt/julia-1.5.3/bin" +``` -To run a scenario which starts at the `1`st hour of the year, runs in `3` -intervals of `24` hours each, loading input data from your present working -directory (`pwd()`) and depositing results in the folder `output`, call: + +#### Julia Package Installation +**Note**: To install the `Gurobi.jl` part of this package, `Julia` will need +to find the Gurobi installation folder. This is done by specifying an environment +variable for `GUROBI_HOME` pointing to the Gurobi ``. + +For more information, see the [Gurobi.jl] documentation. + +As this package is unregistered with Julia, the easiest way to use this package +is to first clone the repo locally (be sure to avoid whitespace in the path): +```bash +git clone https://github.com/Breakthrough-Energy/REISE.jl ``` -REISE.run_scenario(; - interval=24, n_interval=3, start_index=1, outputfolder="output", - inputfolder=pwd()) + +The package will need to be added to each user's default `Julia` environment. +This can be done by opening up `Julia` and activiating the Pkg REPL (the +built-in package manager) with `]`. To exit the Pkg REPL, use `backspace`. + +```julia + _ _ _(_)_ | Documentation: https://docs.julialang.org + (_) | (_) (_) | + _ _ _| |_ __ _ | Type "?" for help, "]?" for Pkg help. + | | | | | | |/ _` | | + | | |_| | | | (_| | | Version 1.5.2 (2020-09-23) + _/ |\__'_|_|_|\__'_| | Official https://julialang.org/ release +|__/ | + +julia> ] + +pkg> ``` -An optional keyword argument `num_segments` controls the linearization of cost -curves into piecewise-linear segments (default is 1). For example: + +From here, there are many ways to add this package to `Julia`. Listed below are +three different options: + +1. `add`ing a package allows a specific branch to be specified from the git +repository. It will use the most recent allowed version of the dependencies +specified in the `Project.toml` file. Currently, this package is known to be +compatible with JuMP v0.21.3; this is specified in the `Project.toml` file, +but there may be other packages for which the latest version does not maintain +backward-compatibility. +```julia +pkg> add /PATH/TO/REISE.jl#develop +``` + +2. `dev`ing a package will always reflect the latest version of the code specified +at the repository. If a branch other than `develop` is checked out, the code in +that branch will be run. Like the above option, this method will also use the +most recent allowed version of the dependencies for which backward-compatibility +is not guaranteed. +```julia +pkg> dev /PATH/TO/REISE.jl ``` + +3. Using the specific environment specified in the project will use the exact +dependency versions specified in the package. This will first have to be +activated and instantiated to download and install all dependencies in `Julia`: + +```julia +pkg> activate /PATH/TO/REISE.jl + Activating environment at `~/REISE.jl/Project.toml` + +(REISE) pkg> instantiate +``` +In order for the below `python` scripts to use this environment, set a +`JULIA_PROJECT` environment variable to the path to `/PATH/TO/REISE.jl`. + +For more information about the different installation options, please see the +documentation for the [Julia Package Manager]. + +#### Verification and Troubleshooting + +To verify that the package has been successfully installed, open a new instance of `Julia` and verify that the `REISE` package can load without any errors with the following command: + +```julia +using REISE +``` + +### Python Packages + +#### Python Requirements Installation +The dependencies of the python scripts contained in `pyreisejl` are not +automatically installed. See `requirements.txt` for details. These requirements +can be installed using pip: +```bash +pip install -r requirements.txt +``` + +#### Python Installation Verification +To verify that the included python scripts can successfully run `REISE`, open a python interpreter and run the following commands. They should return with no errors. +```python +from julia.api import Julia +Julia(compiled_modules=False) +from julia import REISE +``` + +Note that the final import of `REISE` may take a couple of minutes to complete. + +## Usage (Julia) +Installation registers a package named `REISE`. Following Julia naming conventions, the `.jl` is dropped. The package can be imported using: `import REISE` to call `REISE.run_scenario()`, or `using REISE` to call `run_scenario()`. + +Running a scenario requires the following inputs: +- `interval`: the length of each simulation interval (hours). +- `n_interval`: the number of simulation intervals. +- `start_index`: the hour to start the simulation, representing the row of the time-series profiles in `demand.csv`, `hydro.csv`, `solar.csv`, and `wind.csv`. +Note that unlike some other programming languages, Julia is 1-indexed, so the first index is `1`. +- `inputfolder`: the directory from which to load input files. +- `optimizer_factory`: an argument which can be passed to `JuMP.Model` to create a new model instance with an attached solver. +Be sure to pass the factory itself (e.g. `GLPK.Optimizer`) rather than an instance (e.g. `GLPK.Optimizer()`). See the [JuMP.Model documentation] for more information. + +As an example, to run a scenario which starts at the `1`st hour of the year, runs in `3` intervals of `24` hours each, loading input data from your present working directory (`pwd()`), using the `GLPK` solver, call: +```julia +import REISE +import GLPK REISE.run_scenario(; - interval=24, n_interval=3, start_index=1, outputfolder="output", - inputfolder=pwd(), num_segments=3) + interval=24, n_interval=3, start_index=1, inputfolder=pwd(), optimizer_factory=GLPK.Optimizer +) +``` + +Optional arguments include: +- `num_segments`: the number of piecewise linear segments to use when linearizing polynomial cost curves (default is 1). +- `outputfolder`: a directory in which to store results files. The default is a subdirectory `"output"` within the input directory (created if it does not already exist). +- `threads`: the number of threads to be used by the solver. The default is to let the solver decide. +- `solver_kwargs`: a dictionary of `String => value` pairs to be passed to the solver. + +Default settings for running using Gurobi can be accessed if `Gurobi.jl` has already been imported using the `REISE.run_scenario_gurobi` function: +```julia +import REISE +import Gurobi +REISE.run_scenario_gurobi(; + interval=24, n_interval=3, start_index=1, inputfolder=pwd(), +) +``` + +Optional arguments for `REISE.run_scenario` can still be passed as desired. + +## Usage (Python) + +The python scripts included in `pyreisejl` perform some additional input validation for the Julia engine before running the simulation and extract data from the resulting `.mat` files to `.pkl` files. + +There are two main python scripts included in `pyreisejl`: +- `pyreisejl/utility/call.py` +- `pyreisejl/utility/extract_data.py` + +The first of these scripts transforms more descriptive input parameters into the +ones necessary for the Julia engine while also performing some additional input +validation. The latter, which can be set to automatically occur after the +simulation has completed, extracts key metrics from the resulting `.mat` files +to `.pkl` files. + +For example, a simulation can be run as follows: +```bash +pyreisejl/utility/call.py -s '2016-01-01' -e '2016-01-07' -int 24 -i '/PATH/TO/INPUT/FILES' ``` +After the simulation has completed, the extraction can be run using the same start and end date as were used to run the simulation: +```bash +pyreisejl/utility/extract_data.py -s '2016-01-01' -e '2016-01-07' -x '/PATH/TO/OUTPUT/FILES' +``` + + +### Running a Simulation + +**Note** To see the available options for the `call.py` or `extract_data.py` script, use the `-h, --help` flag when calling the script. + +To run the `REISE.jl` simulation from python, using Gurobi as the solver, run `call.py` with the following required options: +``` + -s, --start-date START_DATE + The start date for the simulation in format + 'YYYY-MM-DD'. 'YYYY-MM-DD HH'. 'YYYY-MM-DD HH:MM', + or 'YYYY-MM-DD HH:MM:SS'. + -e, --end-date END_DATE + The end date for the simulation in format + 'YYYY-MM-DD'. 'YYYY-MM-DD HH'. 'YYYY-MM-DD HH:MM', + or 'YYYY-MM-DD HH:MM:SS'. If only the date is specified + (without any hours), the entire end-date will be + included in the simulation. + -int, --interval INTERVAL + The length of each interval in hours. + -i, --input-dir INPUT_DIR + The directory containing the input data files. Required + files are 'case.mat', 'demand.csv', 'hydro.csv', + 'solar.csv', and 'wind.csv'. +``` + +Note that the start and end dates need to match dates contained in the input +profiles (demand, hydro, solar, wind). + + +This python script will validate some of the inputs and translate them into the +required Julia inputs listed below. By default, the Julia engine creates +`result_*.mat` files in an `output` folder created in the given input directory. +To save the matlab files to a different directory, there is an optional flag to +specify the execute directory. If this directory already exists, any existing +computations will be overwritten. +``` + -x EXECUTE_DIR, --execute-dir EXECUTE_DIR + The directory to store the results. This is optional + and defaults to an execute folder that will be created + in the input directory if it does not exist. +``` + +There is another optional flag to specify the number of threads to use for the +simulation run in `Gurobi`. If the number of threads specified is higher than +the number of logical processor count available, the simulation will still run +with a warning. Specifying zero threads defaults to Auto. +``` + -t THREADS, --threads THREADS + The number of threads with which to run the simulation. + This is optional and defaults to Auto. +``` + +The documentation for these options can also been accessed by using the +help flag: +``` + -h, --help show this help message and exit +``` + +### Extracting Simulation Results + +The script `extract_data.py` extracts the following Pandas DataFrames from the +matlab files generated by the Julia engine: + +* PF.pkl (power flow) +* PG.pkl (power generated) +* LMP.pkl (locational marginal price) +* CONGU.pkl (congestion, upper flow limit) +* CONGL.pkl (congestion, lower flow limit) +* AVERAGED_CONG.pkl (time averaged congestion) + +If the grid used in the simulation contains DC lines and/or energy storage devices, the following files will also be extracted as necessary: + +* PF_DCLINE.pkl (power flow on DC lines) +* STORAGE_PG.pkl (power generated by storage units) +* STORAGE_E.pkl (energy state of charge) + +If one or more intervals of the simulation were found to be infeasible without shedding load, the following file will also be extracted: +* LOAD_SHED.pkl (load shed profile for each load bus) + +The extraction process can be memory intensive, so it does not automatically +happen after a simulation run by default. If resource constraints are not a +concern, however, the below flag can be used with `call.py` to automatically +extract the data after a simulation run without having to manually initiate it: + +``` + -d, --extract-data If this flag is used, the data generated by the + simulation after the engine has finished running will be + automatically extracted into .pkl files, and the + result.mat files will be deleted. The extraction process + can be memory intensive. This is optional and defaults + to False if the flag is omitted. +``` + +To manually extract the data, run `extract_data.py` with the following options: + +``` + -s START_DATE, --start-date START_DATE + The start date as provided to run the simulation. + Supported formats are 'YYYY-MM-DD'. 'YYYY-MM-DD HH'. + 'YYYY-MM-DD HH:MM', or 'YYYY-MM-DD HH:MM:SS'. + -e END_DATE, --end-date END_DATE + The end date as provided to run the simulation. + Supported formats are 'YYYY-MM-DD'. 'YYYY-MM-DD HH'. + 'YYYY-MM-DD HH:MM', or 'YYYY-MM-DD HH:MM:SS'. + -x EXECUTE_DIR, --execute-dir EXECUTE_DIR + The directory where the REISE.jl results are stored. +``` + +When manually running the `extract_data` process, the script assumes the +frequency of the input profile csv's are hourly and will construct the +timestamps for the resulting data accordingly. If a different frequency was +used for the input data, it can be specified with the following option: +``` + -f [FREQUENCY], --frequency [FREQUENCY] + The frequency of data points in the original profile + csvs as a Pandas frequency string. This is optional + and defaults to an hour. +``` + +The following optional options are available to both `call.py` when using the +automatic extraction flag and to `extract_data.py`: + +``` + -o OUTPUT_DIR, --output-dir OUTPUT_DIR + The directory to store the extracted data. This is + optional and defaults to the execute directory. + This flag is only used if the extract-data flag is set. + -m MATLAB_DIR, --matlab-dir MATLAB_DIR + The directory to store the modified case.mat used by + the engine. This is optional and defaults to the execute + directory. This flag is only used if the extract-data + flag is set. + -k, --keep-matlab The result.mat files found in the execute directory will + be kept instead of deleted after extraction. This flag + is only used if the extract-data flag is set. +``` + +### Compatibility with PowerSimData + +Within the python code in this repo, there is some code to maintain +compatibility with the `PowerSimData` framework. + +Both `call.py` and `extract_data.py` can be called using a positional +argument that corresponds to a scenario id as generated by the +`PowerSimData` framework. Using this invocation assumes the presence +of the `PowerSimData` infrastructure including both a Scenario List +Manager and Execute List Manager. This option is not intended for manual +simulation runs. + +Note also the different naming convention for various directories by +`PowerSimData` as compared to the options for the python scripts within +this repository. + +## Docker + +The easiest way to setup this engine is within a Docker image. Note, however, that the Docker image is currently configured to use a [Gurobi Cloud License] and not any of the other `Gurobi` licensing options. + +There is an included `Dockerfile` that can be used to build the Docker image. With the Docker daemon installed and running, navigate to the `REISE.jl` folder containing the `Dockerfile` and build the image: + +```bash +docker build . -t reisejl +``` + +To run the Docker image, you will need to mount two volumes; one containing the +`Gurobi` license file and another containing the necessary input files for the +engine. + +```bash +docker run -it -v /LOCAL/PATH/TO/GUROBI.LIC:/usr/share/gurobi_license -v /LOCAL/PATH/TO/DATA:/usr/share/data reisejl bash +``` + +The following command will start a bash shell session within the container, +using the `python` commands described above. + +```bash +python pyreisejl/utility/call.py -s '2016-01-01' -e '2016-01-07' -int 24 -i '/usr/share/data' +``` + +Note that loading the `REISE.jl` package can take up to a couple of minutes, +so there may not be any output in this time. + + ## Package Structure +`REISE.jl` contains only imports and includes. Individual type and function definitions are all in the other files in the `src` folder. -`REISE.jl` contains only imports and includes. Individual type and function -definitions are all in the other files in the `src` folder. ## Formulation - [comment]: # (getting Github to display LaTeX via the approach in https://gist.github.com/a-rodin/fef3f543412d6e1ec5b6cf55bf197d7b) [comment]: # (Encoding LaTeX via https://www.urlencoder.org/) -### Sets -- ![B](https://render.githubusercontent.com/render/math?math=B): +### Sets +- ![B](https://render.githubusercontent.com/render/math?math=B): Set of buses, indexed by ![b](https://render.githubusercontent.com/render/math?math=b). -- ![I](https://render.githubusercontent.com/render/math?math=I): +- ![I](https://render.githubusercontent.com/render/math?math=I): Set of generators, indexed by ![i](https://render.githubusercontent.com/render/math?math=i). -- ![L](https://render.githubusercontent.com/render/math?math=L): +- ![L](https://render.githubusercontent.com/render/math?math=L): Set of transmission network branches, indexed by ![l](https://render.githubusercontent.com/render/math?math=l). -- ![S](https://render.githubusercontent.com/render/math?math=S): +- ![S](https://render.githubusercontent.com/render/math?math=S): Set of generation cost curve segments, indexed by ![s](https://render.githubusercontent.com/render/math?math=s). -- ![T](https://render.githubusercontent.com/render/math?math=T): +- ![T](https://render.githubusercontent.com/render/math?math=T): Set of time periods, indexed by ![t](https://render.githubusercontent.com/render/math?math=t). -#### Subsets +#### Subsets - ![I^{\text{H}}](https://render.githubusercontent.com/render/math?math=I%5E%7B%5Ctext%7BH%7D%7D): Set of hydro generators. - ![I^{\text{S}}](https://render.githubusercontent.com/render/math?math=I%5E%7B%5Ctext%7BS%7D%7D): @@ -110,15 +529,15 @@ Set of solar generators. - ![I^{\text{W}}](https://render.githubusercontent.com/render/math?math=I%5E%7B%5Ctext%7BW%7D%7D): Set of wind generators. -### Variables +### Variables - ![E_{b,t}](https://render.githubusercontent.com/render/math?math=E_%7Bb%2Ct%7D): Energy available in energy storage devices at bus ![b](https://render.githubusercontent.com/render/math?math=b) at time ![t](https://render.githubusercontent.com/render/math?math=t). - ![f_{l,t}](https://render.githubusercontent.com/render/math?math=f_%7Bl%2Ct%7D): Power flowing on branch ![l](https://render.githubusercontent.com/render/math?math=l) at time ![t](https://render.githubusercontent.com/render/math?math=t). -- ![g_{i,t}](https://render.githubusercontent.com/render/math?math=g_%7Bi%2Ct%7D): +- ![g_{i,t}](https://render.githubusercontent.com/render/math?math=g_%7Bi%2Ct%7D): Power injected by each generator ![i](https://render.githubusercontent.com/render/math?math=i) at time ![t](https://render.githubusercontent.com/render/math?math=t). - ![g_{i,s,t}](https://render.githubusercontent.com/render/math?math=g_%7Bi%2Cs%2Ct%7D): @@ -141,8 +560,8 @@ at time ![t](https://render.githubusercontent.com/render/math?math=t). Voltage angle of bus ![b](https://render.githubusercontent.com/render/math?math=b) at time ![t](https://render.githubusercontent.com/render/math?math=t). -### Parameters +### Parameters - ![a^{\text{shed}}](https://render.githubusercontent.com/render/math?math=a%5E%7B%5Ctext%7Bshed%7D%7D): Binary parameter, whether load shedding is enabled. - ![a^{\text{viol}}](https://render.githubusercontent.com/render/math?math=a%5E%7B%5Ctext%7Bviol%7D%7D): @@ -169,7 +588,7 @@ Generator cost curve segment width. - ![J_{b}^{\text{max}}](https://render.githubusercontent.com/render/math?math=J_%7Bb%7D%5E%7B%5Ctext%7Bmax%7D%7D): Maximum charging/discharging power of energy storage devices at bus ![b](https://render.githubusercontent.com/render/math?math=b). - ![m_{l,b}^{\text{line}}](https://render.githubusercontent.com/render/math?math=m_%7Bl%2Cb%7D%5E%7B%5Ctext%7Bline%7D%7D): -Mapping of branches to buses. +Mapping of branches to buses. ![m_{l,b}^{\text{line}} = 1](https://render.githubusercontent.com/render/math?math=m_%7Bl%2Cb%7D%5E%7B%5Ctext%7Bline%7D%7D%20%3D%201) if branch ![l](https://render.githubusercontent.com/render/math?math=l) 'starts' at bus ![b](https://render.githubusercontent.com/render/math?math=b), @@ -258,3 +677,20 @@ Penalty for load shedding (if load shedding is enabled). Penalty for transmission line limit violations (if transmission violations are enabled). - ![p^{\text{e}} \sum_{\b \in B} [E_{b,0} - E_{b,|T|}]](https://render.githubusercontent.com/render/math?math=p%5E%7B%5Ctext%7Be%7D%7D%20%5Csum_%7B%5Cb%20%5Cin%20B%7D%20%5BE_%7Bb%2C0%7D%20-%20E_%7Bb%2C%7CT%7C%7D%5D): Penalty for ending the interval with less stored energy than the start, or reward for ending with more. + +[Gurobi]: https://www.gurobi.com +[Gurobi Installation Guide]: https://www.gurobi.com/documentation/quickstart.html +[Gurobi license file]: https://www.gurobi.com/downloads/ +[Gurobi Cloud license]: https://cloud.gurobi.com/manager/licenses +[Free licenses for academic use]: https://www.gurobi.com/academia/academic-program-and-licenses/ +[Julia]: https://julialang.org/ +[Download Julia]: https://julialang.org/downloads/#current_stable_release +[Python]: https://www.python.org/ +[Download Python]: https://www.python.org/downloads/release/python-386/ +[Docker]: https://docs.docker.com/get-docker/ +[Zenodo]: https://zenodo.org/record/3530898 + +[Gurobi.jl]: https://github.com/JuliaOpt/Gurobi.jl#installation +[Julia Package Manager]: https://julialang.github.io/Pkg.jl/v1/managing-packages/ +[JuMP.Model documentation]: https://jump.dev/JuMP.jl/v0.21.1/solvers/#JuMP.Model-Tuple{Any} +[compatible with JuMP]: https://jump.dev/JuMP.jl/stable/installation/#Supported-solvers diff --git a/pyreisejl/__init__.py b/pyreisejl/__init__.py index c61ece7a..bfbcdfba 100644 --- a/pyreisejl/__init__.py +++ b/pyreisejl/__init__.py @@ -1 +1 @@ -__all__ = ['utility'] +__all__ = ["utility"] diff --git a/pyreisejl/utility/__init__.py b/pyreisejl/utility/__init__.py index a0fc62b9..e69de29b 100644 --- a/pyreisejl/utility/__init__.py +++ b/pyreisejl/utility/__init__.py @@ -1 +0,0 @@ -__all__ = ['const', 'extract_data'] diff --git a/pyreisejl/utility/app.py b/pyreisejl/utility/app.py new file mode 100644 index 00000000..b1d0feef --- /dev/null +++ b/pyreisejl/utility/app.py @@ -0,0 +1,55 @@ +from pathlib import Path +from subprocess import PIPE, Popen + +from flask import Flask, jsonify, request + +from pyreisejl.utility.state import ApplicationState, SimulationState + +app = Flask(__name__) + + +""" +Example request: + +curl -XPOST http://localhost:5000/launch/1234 +curl -XPOST http://localhost:5000/launch/1234?threads=42 +curl http://localhost:5000/status/1234 +""" + + +state = ApplicationState() + + +def get_script_path(): + script_dir = Path(__file__).parent.absolute() + path_to_script = Path(script_dir, "call.py") + return str(path_to_script) + + +@app.route("/launch/", methods=["POST"]) +def launch_simulation(scenario_id): + cmd_call = ["python3", "-u", get_script_path(), str(scenario_id), "--extract-data"] + threads = request.args.get("threads", None) + + if threads is not None: + cmd_call.extend(["--threads", str(threads)]) + + proc = Popen(cmd_call, stdout=PIPE, stderr=PIPE, start_new_session=True) + entry = SimulationState(scenario_id, proc) + state.add(entry) + return jsonify(entry.as_dict()) + + +@app.route("/list") +def list_ongoing(): + return jsonify(state.as_dict()) + + +@app.route("/status/") +def get_status(scenario_id): + entry = state.get(scenario_id) + return jsonify(entry), 200 if entry is not None else 404 + + +if __name__ == "__main__": + app.run(port=5000, debug=True) diff --git a/pyreisejl/utility/call.py b/pyreisejl/utility/call.py index 152ae643..c9178add 100644 --- a/pyreisejl/utility/call.py +++ b/pyreisejl/utility/call.py @@ -1,130 +1,201 @@ -from pyreisejl.utility import const -from pyreisejl.utility.helpers import sec2hms - -import numpy as np import os -import pandas as pd - -from collections import OrderedDict -from multiprocessing import Process from time import time +import pandas as pd -def get_scenario(scenario_id): - """Returns scenario information. - - :param str scenario_id: scenario index. - :return: (*dict*) -- scenario information. - """ - scenario_list = pd.read_csv(const.SCENARIO_LIST, dtype=str) - scenario_list.fillna('', inplace=True) - scenario = scenario_list[scenario_list.id == scenario_id] - - return scenario.to_dict('records', into=OrderedDict)[0] - +from pyreisejl.utility import const, parser +from pyreisejl.utility.extract_data import extract_scenario +from pyreisejl.utility.helpers import ( + InvalidDateArgument, + InvalidInterval, + WrongNumberOfArguments, + extract_date_limits, + get_scenario, + insert_in_file, + sec2hms, + validate_time_format, + validate_time_range, +) -def insert_in_file(filename, scenario_id, column_number, column_value): - """Updates status in execute list on server. - :param str filename: path to execute or scenario list. - :param str scenario_id: scenario index. - :param str column_number: id of column (indexing starts at 1). - :param str column_value: value to insert. - """ - options = "-F, -v OFS=',' -v INPLACE_SUFFIX=.bak -i inplace" - program = ("'{for(i=1; i<=NF; i++){if($1==%s) $%s=\"%s\"}};1'" % - (scenario_id, column_number, column_value)) - command = "awk %s %s %s" % (options, program, filename) - os.system(command) - - -def launch_scenario_performance(scenario_id, n_parallel_call=1): - """Launches the scenario. +def _record_scenario(scenario_id, runtime): + """Updates execute and scenario list on server after simulation. :param str scenario_id: scenario index. - :param int n_parallel_call: number of parallel runs. This function calls - :func:scenario_julia_call. + :param int runtime: runtime of simulation in seconds. """ - scenario_info = get_scenario(scenario_id) - - min_ts = pd.Timestamp('2016-01-01 00:00:00') - max_ts = pd.Timestamp('2016-12-31 23:00:00') - dates = pd.date_range(start=min_ts, end=max_ts, freq='1H') - - start_ts = pd.Timestamp(scenario_info['start_date']) - end_ts = pd.Timestamp(scenario_info['end_date']) - - # Julia starts at 1 - start_index = dates.get_loc(start_ts) + 1 - end_index = dates.get_loc(end_ts) + 1 - - # Create save data folder if does not exist - output_dir = os.path.join(const.EXECUTE_DIR, - 'scenario_%s/output' % scenario_info['id']) - if not os.path.exists(output_dir): - os.mkdir(output_dir) - # Update status in ExecuteList.csv on server - insert_in_file(const.EXECUTE_LIST, scenario_info['id'], '2', 'running') - - # Split the index into n_parallel_call parts - parallel_call_list = np.array_split(range(start_index, end_index + 1), - n_parallel_call) - proc = [] - start = time() - for i in parallel_call_list: - p = Process(target=scenario_julia_call, - args=(scenario_info, int(i[0]), int(i[-1]),)) - p.start() - proc.append(p) - for p in proc: - p.join() - end = time() + insert_in_file(const.EXECUTE_LIST, scenario_id, "status", "finished") - # Update status in ExecuteList.csv on server - insert_in_file(const.EXECUTE_LIST, scenario_info['id'], '2', 'finished') - - runtime = round(end - start) - print('Run time: %s' % str(runtime)) hours, minutes, seconds = sec2hms(runtime) - insert_in_file(const.SCENARIO_LIST, scenario_info['id'], '15', - '%d:%02d' % (hours, minutes)) - - -def scenario_julia_call(scenario_info, start_index, end_index): - """ - Starts a Julia engine, runs the add_path file to load Julia code. - Then, loads the data path and runs the scenario. - - :param dict scenario_info: scenario information. - :param int start_index: start index. - :param int end_index: end index. + insert_in_file( + const.SCENARIO_LIST, scenario_id, "runtime", "%d:%02d" % (hours, minutes) + ) + + +class Launcher: + """Parent class for solver-specific scenario launchers. + + :param str start_date: start date of simulation as 'YYYY-MM-DD HH:MM:SS', + where HH, MM, and SS are optional. + :param str end_date: end date of simulation as 'YYYY-MM-DD HH:MM:SS', + where HH, MM, and SS are optional. + :param int interval: length of each interval in hours + :param str input_dir: directory with input data + :raises InvalidDateArgument: if start_date is posterior to end_date + :raises InvalidInterval: if the interval doesn't evently divide the given date range """ - from julia.api import Julia - jl = Julia(compiled_modules=False) - from julia import Main - from julia import REISE - - interval = int(scenario_info['interval'].split('H', 1)[0]) - n_interval = int((end_index - start_index + 1) / interval) - - input_dir = os.path.join(const.EXECUTE_DIR, - 'scenario_%s' % scenario_info['id']) - output_dir = os.path.join(const.EXECUTE_DIR, - 'scenario_%s/output/' % scenario_info['id']) - - REISE.run_scenario( - interval=interval, - n_interval=n_interval, - start_index=start_index, - inputfolder=input_dir, - outputfolder=output_dir) - Main.eval('exit()') + def __init__(self, start_date, end_date, interval, input_dir): + """Constructor.""" + # extract time limits from 'demand.csv' + with open(os.path.join(input_dir, "demand.csv")) as profile: + min_ts, max_ts, freq = extract_date_limits(profile) + + dates = pd.date_range(start=min_ts, end=max_ts, freq=freq) + + start_ts = validate_time_format(start_date) + end_ts = validate_time_format(end_date, end_date=True) + + # make sure the dates are within the time frame we have data for + validate_time_range(start_ts, min_ts, max_ts) + validate_time_range(end_ts, min_ts, max_ts) + + if start_ts > end_ts: + raise InvalidDateArgument( + f"The start date ({start_ts}) cannot be after the end date ({end_ts})." + ) + + # Julia starts at 1 + start_index = dates.get_loc(start_ts) + 1 + end_index = dates.get_loc(end_ts) + 1 + + # Calculate number of intervals + ts_range = end_index - start_index + 1 + if ts_range % interval > 0: + raise InvalidInterval( + "This interval does not evenly divide the given date range." + ) + self.start_index = start_index + self.interval = interval + self.n_interval = int(ts_range / interval) + self.input_dir = input_dir + print("Validation complete!") + + def _print_settings(self): + print("Launching scenario with parameters:") + print( + { + "interval": self.interval, + "n_interval": self.n_interval, + "start_index": self.start_index, + "input_dir": self.input_dir, + "execute_dir": self.execute_dir, + "threads": self.threads, + } + ) + + def launch_scenario(self): + # This should be defined in sub-classes + raise NotImplementedError + + +class GurobiLauncher(Launcher): + def launch_scenario(self, execute_dir=None, threads=None, solver_kwargs=None): + """Launches the scenario. + + :param None/str execute_dir: directory for execute data. None defaults to an + execute folder that will be created in the input directory + :param None/int threads: number of threads to use. + :param None/dict solver_kwargs: keyword arguments to pass to solver (if any). + :return: (*int*) runtime of scenario in seconds + """ + self.execute_dir = execute_dir + self.threads = threads + self._print_settings() + # Import these within function because there is a lengthy compilation step + from julia.api import Julia + + Julia(compiled_modules=False) + from julia import Gurobi # noqa: F401 + from julia import REISE + + start = time() + REISE.run_scenario_gurobi( + interval=self.interval, + n_interval=self.n_interval, + start_index=self.start_index, + inputfolder=self.input_dir, + outputfolder=self.execute_dir, + threads=self.threads, + ) + end = time() + + runtime = round(end - start) + hours, minutes, seconds = sec2hms(runtime) + print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}") + + return runtime + + +def main(args): + # Get scenario info if using PowerSimData + if args.scenario_id: + scenario_args = get_scenario(args.scenario_id) + + args.start_date = scenario_args[0] + args.end_date = scenario_args[1] + args.interval = scenario_args[2] + args.input_dir = scenario_args[3] + args.execute_dir = scenario_args[4] + + # Update status in ExecuteList.csv on server + insert_in_file(const.EXECUTE_LIST, args.scenario_id, "status", "running") + + # Check to make sure all necessary arguments are there + # (start_date, end_date, interval, input_dir) + if not (args.start_date and args.end_date and args.interval and args.input_dir): + err_str = ( + "The following arguments are required: " + "start-date, end-date, interval, input-dir" + ) + raise WrongNumberOfArguments(err_str) + + launcher = GurobiLauncher( + args.start_date, + args.end_date, + args.interval, + args.input_dir, + ) + runtime = launcher.launch_scenario(args.execute_dir, args.threads) + + # If using PowerSimData, record the runtime + if args.scenario_id: + _record_scenario(args.scenario_id, runtime) + args.matlab_dir = const.INPUT_DIR + args.output_dir = const.OUTPUT_DIR + + if args.extract_data: + if not args.execute_dir: + args.execute_dir = os.path.join(args.input_dir, "output") + + extract_scenario( + args.execute_dir, + args.start_date, + args.end_date, + scenario_id=args.scenario_id, + output_dir=args.output_dir, + mat_dir=args.matlab_dir, + keep_mat=args.keep_matlab, + ) if __name__ == "__main__": - import sys - - launch_scenario_performance(sys.argv[1]) + args = parser.parse_call_args() + try: + main(args) + except Exception as ex: + print(ex) # sent to redirected stdout/stderr + if args.scenario_id: + insert_in_file(const.EXECUTE_LIST, args.scenario_id, "status", "failed") diff --git a/pyreisejl/utility/const.py b/pyreisejl/utility/const.py index d652c1f7..32bbb197 100644 --- a/pyreisejl/utility/const.py +++ b/pyreisejl/utility/const.py @@ -1,5 +1,9 @@ -SCENARIO_LIST = '/home/EGM/v2/ScenarioList.csv' -EXECUTE_LIST = '/home/EGM/v2/ExecuteList.csv' -EXECUTE_DIR = '/home/EGM/v2/tmp' -INPUT_DIR = '/home/EGM/v2/data/input' -OUTPUT_DIR = '/home/EGM/v2/data/output' +import posixpath + +DATA_ROOT_DIR = "/mnt/bes/pcm" + +SCENARIO_LIST = posixpath.join(DATA_ROOT_DIR, "ScenarioList.csv") +EXECUTE_LIST = posixpath.join(DATA_ROOT_DIR, "ExecuteList.csv") +EXECUTE_DIR = posixpath.join(DATA_ROOT_DIR, "tmp") +INPUT_DIR = posixpath.join(DATA_ROOT_DIR, "data", "input") +OUTPUT_DIR = posixpath.join(DATA_ROOT_DIR, "data", "output") diff --git a/pyreisejl/utility/extract_data.py b/pyreisejl/utility/extract_data.py index 768c7858..9851b25a 100644 --- a/pyreisejl/utility/extract_data.py +++ b/pyreisejl/utility/extract_data.py @@ -1,9 +1,6 @@ -from collections import OrderedDict -import datetime as dt import glob -import io import os -import subprocess +import re import time import numpy as np @@ -11,195 +8,142 @@ from scipy.io import loadmat, savemat from tqdm import tqdm -from pyreisejl.utility import const -from pyreisejl.utility.helpers import load_mat73 +from pyreisejl.utility import const, parser +from pyreisejl.utility.helpers import ( + WrongNumberOfArguments, + get_scenario, + insert_in_file, + load_mat73, + validate_time_format, +) -def get_scenario(scenario_id): - """Returns scenario information. +def copy_input(execute_dir, mat_dir=None, scenario_id=None): + """Copies Julia-saved input matfile (input.mat), converting matfile from v7.3 to v7 on the way. - :param str scenario_id: scenario index. - :return: (*dict*) -- scenario information. + :param str execute_dir: the directory containing the original input file + :param str mat_dir: the optional directory to which to save the converted input file, Defaults to execute_dir + :param str filename: optional name for the copied input.mat file. Defaults to "grid.mat" """ - scenario_list = pd.read_csv(const.SCENARIO_LIST, dtype=str) - scenario_list.fillna('', inplace=True) - scenario = scenario_list[scenario_list.id == scenario_id] + if not mat_dir: + mat_dir = execute_dir - return scenario.to_dict('records', into=OrderedDict)[0] + src = os.path.join(execute_dir, "input.mat") + filename = scenario_id + "_grid.mat" if scenario_id else "grid.mat" + dst = os.path.join(mat_dir, filename) + print("loading and parsing input.mat") + input_mpc = load_mat73(src) + print(f"saving converted input.mat as {filename}") + savemat(dst, input_mpc, do_compression=True) -def insert_in_file(filename, scenario_id, column_number, column_value): - """Updates status in execute list on server. + return dst - :param str filename: path to execute or scenario list. - :param str scenario_id: scenario index. - :param str column_number: id of column (indexing starts at 1). - :param str column_value: value to insert. - """ - options = "-F, -v OFS=',' -v INPLACE_SUFFIX=.bak -i inplace" - program = ("'{for(i=1; i<=NF; i++){if($1==%s) $%s=\"%s\"}};1'" % - (scenario_id, column_number, column_value)) - command = "awk %s %s %s" % (options, program, filename) - os.system(command) +def result_num(filename): + """Parses the number out of a filename in the format *result_{number}.mat -def _get_outputs_id(folder): - """Get output id for each applicate output. - - :param str folder: path to folder with input case files. - :return: (*dict*) -- dictionary of {output_name: column_indices} + :param str filename: the filename from which to extract the result number + :return: (*int*) the result number """ - case = loadmat(os.path.join(folder, 'case.mat'), squeeze_me=True, - struct_as_record=False) - - outputs_id = {'pg': case['mpc'].genid, - 'pf': case['mpc'].branchid, - 'lmp': case['mpc'].bus[:, 0].astype(np.int64), - 'load_shed': case['mpc'].bus[:, 0].astype(np.int64), - 'congu': case['mpc'].branchid, - 'congl': case['mpc'].branchid} - try: - outputs_id['pf_dcline'] = case['mpc'].dclineid - except AttributeError: - pass - try: - case_storage = loadmat( - os.path.join(folder, 'case_storage'), squeeze_me=True, - struct_as_record=False) - num_storage = len(case_storage['storage'].gen) - outputs_id['storage_pg'] = np.arange(num_storage) - outputs_id['storage_e'] = np.arange(num_storage) - except FileNotFoundError: - pass + match = re.match(r".*?result_(?P\d+)\.mat$", filename) - return outputs_id + return int(match.group("num")) -def extract_data(scenario_info): +def extract_data(results): """Builds data frames of {PG, PF, LMP, CONGU, CONGL} from Julia simulation output binary files produced by REISE.jl. - :param dict scenario_info: scenario information. - :return: (*pandas.DataFrame*) -- data frames of: - PG, PF, LMP, CONGU, CONGL, LOAD_SHED. + :param list results: list of result files + :return: (*tuple*) -- first element is a dictionary of Pandas data frames of: + PG, PF, LMP, CONGU, CONGL, LOAD_SHED, second is a list of strings of infeasibilities, + and the third element is a list of numpy.float64 costs for each file in the input results list """ + infeasibilities = [] cost = [] - setup_time = [] - solve_time = [] - optimize_time = [] - extraction_vars = {'pf', 'pg', 'lmp', 'congu', 'congl'} - sparse_extraction_vars = {'congu', 'congl', 'load_shed'} + extraction_vars = {"pf", "pg", "lmp", "congu", "congl"} + sparse_extraction_vars = {"congu", "congl", "load_shed"} temps = {} outputs = {} - folder = os.path.join(const.EXECUTE_DIR, - 'scenario_%s' % scenario_info['id']) - end_index = len(glob.glob(os.path.join(folder, 'output', 'result_*.mat'))) - tic = time.process_time() - for i in tqdm(range(end_index)): - filename = 'result_' + str(i) + '.mat' - - output = load_mat73(os.path.join(folder, 'output', filename)) + for i, filename in tqdm(enumerate(results)): + # For each result_#.mat file + output = load_mat73(filename) + # Record cost for this mat file try: - cost.append(output['mdo_save']['results']['f'][0][0]) + cost.append(output["mdo_save"]["results"]["f"][0][0]) except KeyError: pass - demand_scaling = output['mdo_save']['demand_scaling'][0][0] + # Check for infeasibilities + demand_scaling = output["mdo_save"]["demand_scaling"][0][0] if demand_scaling < 1: demand_change = round(100 * (1 - demand_scaling)) - infeasibilities.append('%s:%s' % (str(i), str(demand_change))) - output_mpc = output['mdo_save']['flow']['mpc'] - temps['pg'] = output_mpc['gen']['PG'].T - temps['pf'] = output_mpc['branch']['PF'].T - temps['lmp'] = output_mpc['bus']['LAM_P'].T - temps['congu'] = output_mpc['branch']['MU_SF'].T - temps['congl'] = output_mpc['branch']['MU_ST'].T + infeasibilities.append(f"{i}:{demand_change}") + + # Extract various variables + output_mpc = output["mdo_save"]["flow"]["mpc"] + + temps["pg"] = output_mpc["gen"]["PG"].T + temps["pf"] = output_mpc["branch"]["PF"].T + temps["lmp"] = output_mpc["bus"]["LAM_P"].T + temps["congu"] = output_mpc["branch"]["MU_SF"].T + temps["congl"] = output_mpc["branch"]["MU_ST"].T + + # Extract optional variables (not present in all scenarios) try: - temps['pf_dcline'] = output_mpc['dcline']['PF_dcline'].T - extraction_vars |= {'pf_dcline'} + temps["pf_dcline"] = output_mpc["dcline"]["PF_dcline"].T + extraction_vars |= {"pf_dcline"} except KeyError: pass + try: - temps['storage_pg'] = output_mpc['storage']['PG'].T - temps['storage_e'] = output_mpc['storage']['Energy'].T - extraction_vars |= {'storage_pg', 'storage_e'} + temps["storage_pg"] = output_mpc["storage"]["PG"].T + temps["storage_e"] = output_mpc["storage"]["Energy"].T + extraction_vars |= {"storage_pg", "storage_e"} except KeyError: pass try: - temps['load_shed'] = output_mpc['load_shed']['load_shed'].T - extraction_vars |= {'load_shed'} + temps["load_shed"] = output_mpc["load_shed"]["load_shed"].T + extraction_vars |= {"load_shed"} except KeyError: pass + + # Extract which number result currently being processed + i = result_num(filename) + for v in extraction_vars: + # Determine start, end indices of the outputs where this iteration belongs + interval_length, n_columns = temps[v].shape + start_hour, end_hour = (i * interval_length), ((i + 1) * interval_length) + # If this extraction variables hasn't been seen yet, initialize all zeros if v not in outputs: - interval_length, n_columns = temps[v].shape - total_length = end_index * interval_length + total_length = len(results) * interval_length outputs[v] = pd.DataFrame(np.zeros((total_length, n_columns))) - outputs[v].name = str(scenario_info['id']) + '_' + v.upper() - start_hour, end_hour = (i*interval_length), ((i+1)*interval_length) + # Update the output variables for the time frame with the extracted data outputs[v].iloc[start_hour:end_hour, :] = temps[v] - print(extraction_vars) - + # Record time to read all the data toc = time.process_time() - print('Reading time ' + str(round(toc-tic)) + 's') - - # Write infeasibilities - insert_in_file(const.SCENARIO_LIST, scenario_info['id'], '16', - '_'.join(infeasibilities)) - - # Build log: costs from matfiles, file attributes from ls/awk - log = pd.DataFrame(data={'cost': cost}) - file_filter = os.path.join(folder, 'output', 'result_*.mat') - ls_options = '-lrt --time-style="+%Y-%m-%d %H:%M:%S" ' + file_filter - awk_options = "-v OFS=','" - awk_program = ( - "'BEGIN{print \"filesize,datetime,filename\"}; " - "NR >0 {print $5, $6\" \"$7, $8}'") - ls_call = "ls %s | awk %s %s" % (ls_options, awk_options, awk_program) - ls_output = subprocess.Popen(ls_call, shell=True, stdout=subprocess.PIPE) - utf_ls_output = io.StringIO(ls_output.communicate()[0].decode('utf-8')) - properties_df = pd.read_csv(utf_ls_output, sep=',', dtype=str) - log['filesize'] = properties_df.filesize - log['write_datetime'] = properties_df.datetime - # Write log - log_filename = scenario_info['id'] + '_log.csv' - log.to_csv(os.path.join(const.OUTPUT_DIR, log_filename), header=True) - - # Set index of data frame - date_range = pd.date_range(scenario_info['start_date'], - scenario_info['end_date'], - freq='H') - - for v in extraction_vars: - outputs[v].index = date_range - outputs[v].index.name = 'UTC' - - # Get/set index column name of data frame - outputs_id = _get_outputs_id(folder) - for k in outputs: - index = outputs_id[k] - if isinstance(index, int): - outputs[k].columns = [index] - else: - outputs[k].columns = index.tolist() + print("Reading time " + str((toc - tic)) + "s") - print('converting to float32') - for v in extraction_vars: + # Convert everything except sparse variables to float32 + for v in extraction_vars - sparse_extraction_vars: outputs[v] = outputs[v].astype(np.float32) # Convert outputs with many zero or near-zero values to sparse dtype - to_sparsify = set(extraction_vars) & sparse_extraction_vars - print('sparsifying', to_sparsify) + # As identified in sparse_extraction_vars + to_sparsify = extraction_vars & sparse_extraction_vars + print("sparsifying", to_sparsify) for v in to_sparsify: outputs[v] = outputs[v].round(6).astype(pd.SparseDtype("float", 0)) - return outputs + return outputs, infeasibilities, cost def calculate_averaged_congestion(congl, congu): @@ -213,81 +157,239 @@ def calculate_averaged_congestion(congl, congu): :raises TypeError: if arguments are not data frame. :raises ValueError: if shape or indices of data frames differ. """ + for k, v in locals().items(): if not isinstance(v, pd.DataFrame): - raise TypeError('%s must be a pandas data frame' % k) + raise TypeError(f"{k} must be a pandas data frame") if congl.shape != congu.shape: - raise ValueError('%data frame must have same shape') + raise ValueError("Data frames congu and congl must have same shape") if not all(congl.columns == congu.columns): - raise ValueError('%data frame must have same indices') + raise ValueError("Data frames congu and congl must have same indices") mean_congl = congl.mean() - mean_congl.name = 'CONGL' + mean_congl.name = "CONGL" mean_congu = congu.mean() - mean_congu.name = 'CONGU' + mean_congu.name = "CONGU" return pd.merge(mean_congl, mean_congu, left_index=True, right_index=True) -def copy_input(scenario_id): - """Copies input file, converting matfile from v7.3 to v7 on the way. +def _get_pkl_path(output_dir, scenario_id=None): + """Generates a function to create the path for a .pkl file given - :param str scenario_id: scenario id + :param str output_dir: the directory to save all the .pkl files + :param str scenario_id: optional scenario ID number to prepend to each pickle file. Defaults to None. + :return: (*func*) a function that take a (*str*) attribute name + and returns a (*str*) path to the .pkl where it should be saved """ - src = os.path.join(const.EXECUTE_DIR, - 'scenario_%s' % scenario_id, - 'output', - 'input.mat') - dst = os.path.join(const.INPUT_DIR, - '%s_grid.mat' % scenario_id) - print('loading and parsing input.mat') - input_mpc = load_mat73(src) - print('saving converted input.mat as %s_grid.mat' % scenario_id) - savemat(dst, input_mpc, do_compression=True) + prepend = scenario_id + "_" if scenario_id else "" + + return lambda x: os.path.join(output_dir, prepend + x.upper() + ".pkl") -def delete_output(scenario_id): - """Deletes output MAT-files. +def build_log(mat_results, costs, output_dir, scenario_id=None): + """Build log recording the cost, filesize, and time for each mat file - :param str scenario_id: scenario id. + :param list mat_results: list of filenames for which to log information + :param list costs: list of costs from extract_data corresponding to the mat files + :param str output_dir: directory to save the log file + :param str scenario_id: optional scenario ID number to prepend to the log """ - folder = os.path.join(const.EXECUTE_DIR, - 'scenario_%s' % scenario_id, - 'output') - files = glob.glob(os.path.join(folder, 'result_*.mat')) - for f in files: - os.remove(f) + # Create log name + log_filename = scenario_id + "_log.csv" if scenario_id else "log.csv" -def extract_scenario(scenario_id): - """Extracts data and save data as pickle files. + os.makedirs(output_dir, exist_ok=True) + with open(os.path.join(output_dir, log_filename), "w") as log: + # Write headers + log.write(",cost,filesize,write_datetime\n") + for i in range(len(costs)): + result = mat_results[i] + # Get filesize + filesize = str(os.stat(result).st_size) + # Get formatted ctime + write_datetime = os.stat(result).st_ctime + write_datetime = time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(write_datetime) + ) + + log_vals = [i, costs[i], filesize, write_datetime] + log.write(",".join([str(val) for val in log_vals]) + "\n") + + +def _get_outputs_from_converted(matfile): + """Get output id for each applicate output. - :param str scenario_id: scenario id. + :param dict matfile: dictionary representing the converted input.mat file outputted by REISE.jl + :return: (*dict*) -- dictionary of {output_name: column_indices} """ - scenario_info = get_scenario(scenario_id) + case = matfile["mdi"] + + outputs_id = { + "pg": case.mpc.genid, + "pf": case.mpc.branchid, + "lmp": case.mpc.bus[:, 0].astype(np.int64), + "load_shed": case.mpc.bus[:, 0].astype(np.int64), + "congu": case.mpc.branchid, + "congl": case.mpc.branchid, + } + + try: + outputs_id["pf_dcline"] = case.mpc.dclineid + except AttributeError: + pass + + try: + storage_index = case.Storage.UnitIdx + num_storage = 1 if isinstance(storage_index, float) else len(storage_index) + outputs_id["storage_pg"] = np.arange(num_storage) + outputs_id["storage_e"] = np.arange(num_storage) + except AttributeError: + pass + + _cast_keys_as_lists(outputs_id) + + return outputs_id + + +def _cast_keys_as_lists(dictionary): + """Converts dictionary with values that are ints or numpy arrays to lists. + + :param dict dictionary: dictionary with values that are ints or numpy arrays + :return: (*dict*) -- the same dictionary where the values are lists + """ + for key, value in dictionary.items(): + if type(value) == int: + dictionary[key] = [value] + else: + dictionary[key] = value.tolist() - copy_input(scenario_id) - outputs = extract_data(scenario_info) - print('saving pickles') +def _update_outputs_labels(outputs, start_date, end_date, freq, matfile): + """Updates outputs with the correct date index and column names + + :param dict outputs: dictionary of pandas.DataFrames outputted by extract_data + :param str start_date: start date used for the simulation + :param str end_date: end date used for the simulation + :param str freq: the frequency of timestamps in the input profiles as a pandas frequency alias + :param dict matfile: dictionary representing the converted input.mat file outputted by REISE.jl + """ + # Set index of data frame + start_ts = validate_time_format(start_date) + end_ts = validate_time_format(end_date, end_date=True) + + date_range = pd.date_range(start_ts, end_ts, freq=freq) + + outputs_id = _get_outputs_from_converted(matfile) + + for k in outputs: + outputs[k].index = date_range + outputs[k].index.name = "UTC" + + outputs[k].columns = outputs_id[k] + + +def extract_scenario( + execute_dir, + start_date, + end_date, + scenario_id=None, + output_dir=None, + mat_dir=None, + freq="H", + keep_mat=True, +): + """Extracts data and save data as pickle files to the output directory + + :param str execute_dir: directory containing all of the result.mat files from REISE.jl + :param str start_date: the start date of the simulation run + :param str end_date: the end date of the simulation run + :param str scenario_id: optional identifier for the scenario, used to label output files + :param str output_dir: optional directory in which to store the outputs. defaults to the execute_dir + :param str mat_dir: optional directory in which to store the converted grid.mat file. defaults to the execute_dir + :param bool keep_mat: optional parameter to keep the large result*.mat files after the data has been extracted. Defaults to True. + """ + + # If output or input dir were not specified, default to the execute_dir + output_dir = output_dir or execute_dir + mat_dir = mat_dir or execute_dir + + # Copy input.mat from REISE.jl and convert to .mat v7 for scipy compatibility + converted_mat_path = copy_input(execute_dir, mat_dir, scenario_id) + + # Extract outputs, infeasibilities, cost + mat_results = glob.glob(os.path.join(execute_dir, "result_*.mat")) + mat_results = sorted(mat_results, key=result_num) + + outputs, infeasibilities, cost = extract_data(mat_results) + + # Write log file with costs for each result*.mat file + build_log(mat_results, cost, output_dir, scenario_id) + + # Update outputs with date indices from the copied input.mat file + matfile = loadmat(converted_mat_path, squeeze_me=True, struct_as_record=False) + _update_outputs_labels(outputs, start_date, end_date, freq, matfile) + + # Save pickles + pkl_path = _get_pkl_path(output_dir, scenario_id) + for k, v in outputs.items(): - pickle_filename = scenario_info['id'] + '_' + k.upper() + '.pkl' - v.to_pickle(os.path.join(const.OUTPUT_DIR, pickle_filename)) + v.to_pickle(pkl_path(k.upper())) + + # Calculate and save averaged congestion + calculate_averaged_congestion(outputs["congl"], outputs["congu"]).to_pickle( + pkl_path("AVERAGED_CONG") + ) - calculate_averaged_congestion( - outputs['congl'], outputs['congu']).to_pickle(os.path.join( - const.OUTPUT_DIR, scenario_info['id'] + '_AVERAGED_CONG.pkl')) + if scenario_id: + # Record infeasibilities + insert_in_file( + const.SCENARIO_LIST, + scenario_id, + "infeasibilities", + "_".join(infeasibilities), + ) - insert_in_file(const.EXECUTE_LIST, scenario_info['id'], '2', 'extracted') - insert_in_file(const.SCENARIO_LIST, scenario_info['id'], '4', 'analyze') + # Update execute and scenario list + insert_in_file(const.EXECUTE_LIST, scenario_id, "status", "extracted") + insert_in_file(const.SCENARIO_LIST, scenario_id, "state", "analyze") - print('deleting matfiles') - delete_output(scenario_id) + if not keep_mat: + print("deleting matfiles") + for matfile in mat_results: + os.remove(matfile) if __name__ == "__main__": - import sys - extract_scenario(sys.argv[1]) + args = parser.parse_extract_args() + + # Get scenario info if using PowerSimData + if args.scenario_id: + args.start_date, args.end_date, _, _, args.execute_dir = get_scenario( + args.scenario_id + ) + + args.matlab_dir = const.INPUT_DIR + args.output_dir = const.OUTPUT_DIR + + # Check to make sure all necessary arguments are there + # (start_date, end_date, execute_dir) + if not (args.start_date and args.end_date and args.execute_dir): + err_str = ( + "The following arguments are required: start-date, end-date, execute-dir" + ) + raise WrongNumberOfArguments(err_str) + + extract_scenario( + args.execute_dir, + args.start_date, + args.end_date, + args.scenario_id, + args.output_dir, + args.matlab_dir, + args.frequency, + args.keep_matlab, + ) diff --git a/pyreisejl/utility/helpers.py b/pyreisejl/utility/helpers.py index e96d5dd8..7fe7b356 100644 --- a/pyreisejl/utility/helpers.py +++ b/pyreisejl/utility/helpers.py @@ -1,5 +1,31 @@ +import os +import re +import shutil +from collections import OrderedDict + import h5py import numpy as np +import pandas as pd + +from pyreisejl.utility import const + + +class WrongNumberOfArguments(TypeError): + """To be used when the wrong number of arguments are specified at command line.""" + + pass + + +class InvalidDateArgument(TypeError): + """To be used when an invalid string is passed for the start or end date.""" + + pass + + +class InvalidInterval(TypeError): + """To be used when the interval does not evenly divide the date range given.""" + + pass def sec2hms(seconds): @@ -7,11 +33,11 @@ def sec2hms(seconds): :param int seconds: number of seconds :return: (*tuple*) -- first element is number of hour(s), second is number - od minutes(s) and third is number of second(s) + of minutes(s) and third is number of second(s) :raises TypeError: if argument is not an integer. """ if not isinstance(seconds, int): - raise TypeError('seconds must be an integer') + raise TypeError("seconds must be an integer") minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) @@ -25,7 +51,8 @@ def load_mat73(filename): :param str filename: path to file which will be loaded. :return: (*dict*) -- A possibly nested dictionary of numpy arrays. """ - def convert(path='/'): + + def convert(path="/"): """A recursive walk through the HDF5 structure. :param str path: traverse from where in the HDF5 tree, default is '/'. @@ -34,22 +61,26 @@ def convert(path='/'): output = {} references[path] = output = {} for k, v in f[path].items(): - if type(v).__name__ == 'Group': - output[k] = convert('{path}/{k}'.format(path=path, k=k)) + if type(v).__name__ == "Group": + output[k] = convert("{path}/{k}".format(path=path, k=k)) continue # Retrieve numpy array from h5py_hl.dataset.Dataset data = v[()] - if data.dtype == 'object': + if data.dtype == "object": # Extract values from HDF5 object references original_dims = data.shape data = np.array([f[r][()] for r in data.flat]) # For any entry that is a uint16 array object, convert to str data = np.array( - [''.join([str(c[0]) for c in np.char.mod('%c', array)]) - if array.dtype == np.uint16 else array - for array in data]) + [ + "".join([str(c[0]) for c in np.char.mod("%c", array)]) + if array.dtype == np.uint16 + else array + for array in data + ] + ) # If data is all strs, set dtype to object to save a cell array - if data.dtype.kind in {'U', 'S'}: + if data.dtype.kind in {"U", "S"}: data = np.array(data, dtype=np.object) # Un-flatten arrays which had been flattened if len(original_dims) > 1: @@ -64,5 +95,129 @@ def convert(path='/'): return output references = {} - with h5py.File(filename, 'r') as f: + with h5py.File(filename, "r") as f: return convert() + + +def extract_date_limits(profile_csv): + """Parses a profile csv to extract the first and last time stamp + as well as the time + + :param iterator: iterator containing the data of a profile.csv + :return: (*tuple*) -- (min timestamp, max timestamp, timestamp frequency) as pandas.Timestamp + """ + + profile = pd.read_csv(profile_csv, index_col=0, parse_dates=True) + min_ts = profile.index.min() + max_ts = profile.index.max() + freq = pd.infer_freq(profile.index) + + return (min_ts, max_ts, freq) + + +def validate_time_format(date, end_date=False): + """Validates that the given dates are valid, + and adds 23 hours if an end date is specified without hours. + + :param str date: date string as 'YYYY-MM-DD HH:MM:SS', + where HH, MM, and SS are optional. + :param bool end_date: whether or not this date is an end date + :return: (*pandas.Timestamp*) -- the valid date as a pandas timestamp + :raises InvalidDateArgument: if the date given is not one of the accepted formats + """ + regex = r"^\d{4}-\d{1,2}-\d{1,2}( (?P\d{1,2})(:\d{1,2})?(:\d{1,2})?)?$" + match = re.match(regex, date) + + if match: + # if pandas won't convert the regex match, it's not a valid date + # (i.e. invalid month or date) + try: + valid_date = pd.Timestamp(date) + except ValueError: + raise InvalidDateArgument(f"{date} is not a valid timestamp.") + + # if an end_date is given with no hours, + # assume date range is until the end of the day (23h) + if end_date and not match.group("hour"): + valid_date += pd.Timedelta(hours=23) + + else: + err_str = f"'{date}' is an invalid date. It needs to be in the form YYYY-MM-DD." + raise InvalidDateArgument(err_str) + + return valid_date + + +def validate_time_range(date, min_ts, max_ts): + """Validates that a date is within the given time range. + + :param pandas.Timestamp date: date to validate + :param pandas.Timestamp date: start date of time range + :param pandas.Timestamp date: end date of time range + :raises InvalidDateArgument: if the date is not between + the minimum and maximum timestamps + """ + # make sure the dates are within the time frame we have data for + if date < min_ts or date > max_ts: + err_str = f"'{date}' is an invalid date. Valid dates are between {min_ts} and {max_ts}." + raise InvalidDateArgument(err_str) + + +def get_scenario(scenario_id): + """Returns scenario information. + + :param int/str scenario_id: scenario index. + :return: (*tuple*) -- scenario start_date, end date, interval, input_dir, execute_dir + """ + # Parses scenario info out of scenario list + scenario_list = pd.read_csv(const.SCENARIO_LIST, dtype=str) + scenario_list.fillna("", inplace=True) + scenario = scenario_list[scenario_list.id == str(scenario_id)] + scenario_info = scenario.to_dict("records", into=OrderedDict)[0] + + # Determine input and execute directory for data + input_dir = os.path.join(const.EXECUTE_DIR, "scenario_%s" % scenario_info["id"]) + execute_dir = os.path.join( + const.EXECUTE_DIR, f"scenario_{str(scenario_id)}", "output" + ) + + # Grab start and end date for scenario + start_date = scenario_info["start_date"] + end_date = scenario_info["end_date"] + + # Grab interval for scenario + interval = int(scenario_info["interval"].split("H", 1)[0]) + + return start_date, end_date, interval, input_dir, execute_dir + + +def insert_in_file(filename, scenario_id, column_name, column_value): + """Updates status in execute list on server. + + :param str filename: path to execute or scenario list. + :param int/str scenario_id: scenario index. + :param str column_name: name of column to modify. + :param str column_value: value to insert. + """ + _ = shutil.copyfile(filename, filename + ".bak") + + table = pd.read_csv(filename, dtype=str) + table.set_index("id", inplace=True) + table.loc[str(scenario_id), column_name] = column_value + table.to_csv(filename) + + +def get_scenario_status(scenario_id): + """Get the status of the scenario. + + :param int scenario_id: scenario index + :return: (*str*) -- the status, e.g. running, finished, etc, or None + """ + try: + table = pd.read_csv(const.EXECUTE_LIST, index_col="id") + return table.loc[scenario_id, "status"] + except KeyError: + return None + except Exception as ex: + print("Failed to read execute list. It's likely the file does not exist.") + print(f"Exception message: {ex}") diff --git a/pyreisejl/utility/parser.py b/pyreisejl/utility/parser.py new file mode 100644 index 00000000..fe973cc3 --- /dev/null +++ b/pyreisejl/utility/parser.py @@ -0,0 +1,150 @@ +import argparse + + +def parse_call_args(): + parser = argparse.ArgumentParser(description="Run REISE.jl simulation.") + + # Arguments needed to run REISE.jl + parser.add_argument( + "-s", + "--start-date", + help="The start date for the simulation in format 'YYYY-MM-DD', 'YYYY-MM-DD HH', " + "'YYYY-MM-DD HH:MM', or 'YYYY-MM-DD HH:MM:SS'.", + ) + parser.add_argument( + "-e", + "--end-date", + help="The end date for the simulation in format 'YYYY-MM-DD', 'YYYY-MM-DD HH', " + "'YYYY-MM-DD HH:MM', or 'YYYY-MM-DD HH:MM:SS'. If only the date is specified " + "(without any hours), the entire end-date will be included in the simulation.", + ) + parser.add_argument( + "-int", "--interval", help="The length of each interval in hours.", type=int + ) + parser.add_argument( + "-i", + "--input-dir", + help="The directory containing the input data files. " + "Required files are 'case.mat', 'demand.csv', " + "'hydro.csv', 'solar.csv', and 'wind.csv'.", + ) + parser.add_argument( + "-x", + "--execute-dir", + help="The directory to store the results. This is optional and defaults " + "to an execute folder that will be created in the input directory " + "if it does not exist.", + ) + parser.add_argument( + "-t", + "--threads", + type=int, + help="The number of threads to run the simulation with. " + "This is optional and defaults to Auto.", + ) + parser.add_argument( + "-d", + "--extract-data", + action="store_true", + help="If this flag is used, the data generated by the simulation after the engine " + "has finished running will be automatically extracted into .pkl files, " + "and the result.mat files will be deleted. " + "The extraction process can be memory intensive. " + "This is optional and defaults to False if the flag is omitted.", + ) + parser.add_argument( + "-o", + "--output-dir", + help="The directory to store the extracted data. This is optional and defaults " + "to the execute directory. This flag is only used if the extract-data flag is set.", + ) + parser.add_argument( + "-m", + "--matlab-dir", + help="The directory to store the modified case.mat used by the engine. " + "This is optional and defaults to the execute directory. " + "This flag is only used if the extract-data flag is set.", + ) + parser.add_argument( + "-k", + "--keep-matlab", + action="store_true", + help="The result.mat files found in the execute directory will be kept " + "instead of deleted after extraction. " + "This flag is only used if the extract-data flag is set.", + ) + + # For backwards compatability with PowerSimData + parser.add_argument( + "scenario_id", + nargs="?", + default=None, + help="Scenario ID only if using PowerSimData. ", + ) + return parser.parse_args() + + +def parse_extract_args(): + parser = argparse.ArgumentParser( + description="Extract data from the results of the REISE.jl simulation." + ) + + # Arguments needed to run REISE.jl + parser.add_argument( + "-s", + "--start-date", + help="The start date as provided to run the simulation. Supported formats are" + " 'YYYY-MM-DD', 'YYYY-MM-DD HH', 'YYYY-MM-DD HH:MM', or 'YYYY-MM-DD HH:MM:SS'.", + ) + parser.add_argument( + "-e", + "--end-date", + help="The end date as provided to run the simulation. Supported formats are" + " 'YYYY-MM-DD', 'YYYY-MM-DD HH', 'YYYY-MM-DD HH:MM', or 'YYYY-MM-DD HH:MM:SS'.", + ) + parser.add_argument( + "-x", + "--execute-dir", + help="The directory where the REISE.jl results are stored.", + ) + parser.add_argument( + "-o", + "--output-dir", + nargs="?", + default=None, + help="The directory to store the results. This is optional and defaults " + "to the execute directory.", + ) + parser.add_argument( + "-m", + "--matlab-dir", + nargs="?", + default=None, + help="The directory to store the modified case.mat used by the engine. " + "This is optional and defaults to the execute directory.", + ) + parser.add_argument( + "-f", + "--frequency", + nargs="?", + default="H", + help="The frequency of data points in the original profile csvs as a " + "Pandas frequency string. " + "This is optional and defaults to an hour.", + ) + parser.add_argument( + "-k", + "--keep-matlab", + action="store_true", + help="If this flag is used, the result.mat files found in the " + "execute directory will be kept instead of deleted.", + ) + + # For backwards compatability with PowerSimData + parser.add_argument( + "scenario_id", + nargs="?", + default=None, + help="Scenario ID only if using PowerSimData.", + ) + return parser.parse_args() diff --git a/pyreisejl/utility/state.py b/pyreisejl/utility/state.py new file mode 100644 index 00000000..6e6a824a --- /dev/null +++ b/pyreisejl/utility/state.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass, field +from queue import Empty, Queue +from threading import Thread +from typing import Any, Dict, List + +from pyreisejl.utility.helpers import get_scenario_status + + +class Listener: + """Runs in the background to read from stdout/stderr of a long lived + process""" + + def __init__(self, stream): + self.stream = stream + self.queue = Queue() + self._start() + + def _start(self): + t = Thread(target=self._enqueue_output) + t.daemon = True + t.start() + + def _enqueue_output(self): + for line in self.stream: + s = line.decode().strip() + if len(s) > 0: + self.queue.put(s) + self.stream.close() + + def poll(self): + """Get the latest output from the stream + + :return: (*list*) -- list of lines since previous poll + """ + result = [] + try: + while True: + line = self.queue.get_nowait() + result.append(line) + except Empty: # noqa + pass + return result + + +@dataclass +class SimulationState: + """Track the state of an ongoing simulation""" + + _EXCLUDE = ["proc", "out_listener", "err_listener"] + + scenario_id: int + proc: Any = field(default=None, repr=False, compare=False, hash=False) + output: List = field(default_factory=list, repr=False, compare=False, hash=False) + errors: List = field(default_factory=list, repr=False, compare=False, hash=False) + status: str = None + + def __post_init__(self): + self.out_listener = Listener(self.proc.stdout) + self.err_listener = Listener(self.proc.stderr) + + def _refresh(self): + """Set the latest status and append the latest output from standard + streams. + """ + self.status = get_scenario_status(self.scenario_id) + self.output += self.out_listener.poll() + self.errors += self.err_listener.poll() + + def as_dict(self): + """Return custom dict which omits the process attribute which is not + serializable. + + :return: (*dict*) -- dict of the instance attributes + """ + self._refresh() + return {k: v for k, v in self.__dict__.items() if k not in self._EXCLUDE} + + +@dataclass +class ApplicationState: + """Tracks all simulations during the lifetime of the application""" + + ongoing: Dict[int, SimulationState] = field(default_factory=dict) + + def add(self, entry): + """Add entry for scenario to current state + + :param SimulationState entry: object to track a given scenario + """ + self.ongoing[int(entry.scenario_id)] = entry + + def get(self, scenario_id): + """Get the latest information for a scenario if it is present + + :param int scenario_id: id of the scenario + :return: (*dict*) -- a dict containing values from the ScenarioState + """ + if scenario_id not in self.ongoing: + return None + return self.ongoing[scenario_id].as_dict() + + def as_dict(self): + """Custom dict implementation which utilizes the similar method from + SimulationState + + :return: (*dict*) -- dict of the instance attributes + """ + return {k: v.as_dict() for k, v in self.ongoing.items()} diff --git a/pyreisejl/utility/tests/__init__.py b/pyreisejl/utility/tests/__init__.py index a1dc2ca6..e69de29b 100644 --- a/pyreisejl/utility/tests/__init__.py +++ b/pyreisejl/utility/tests/__init__.py @@ -1 +0,0 @@ -__all__ = ['test_extract_data'] diff --git a/pyreisejl/utility/tests/test_call.py b/pyreisejl/utility/tests/test_call.py index 2730faaa..d441d0f1 100644 --- a/pyreisejl/utility/tests/test_call.py +++ b/pyreisejl/utility/tests/test_call.py @@ -4,4 +4,5 @@ @pytest.mark.skip(reason="Need to run on the server") def test(): from pyreisejl.utility.call import launch_scenario_performance - launch_scenario_performance('87') + + launch_scenario_performance("87") diff --git a/pyreisejl/utility/tests/test_extract_data.py b/pyreisejl/utility/tests/test_extract_data.py index c157d827..cbec6ad6 100644 --- a/pyreisejl/utility/tests/test_extract_data.py +++ b/pyreisejl/utility/tests/test_extract_data.py @@ -2,7 +2,12 @@ import pandas as pd import pytest -from ..extract_data import calculate_averaged_congestion +from pyreisejl.utility.extract_data import ( + _cast_keys_as_lists, + _get_pkl_path, + calculate_averaged_congestion, + result_num, +) def test_calculate_averaged_congestion_first_arg_type(): @@ -20,41 +25,65 @@ def test_calculate_averaged_congestion_second_arg_type(): def test_calculate_averaged_congestion_args_shape(): - congl = pd.DataFrame({'A': [1, 2, 3], 'B': [10, 11, 12]}) - congu = pd.DataFrame({'A': [21, 22, 23, 24], 'B': [30, 31, 32, 33]}) + congl = pd.DataFrame({"A": [1, 2, 3], "B": [10, 11, 12]}) + congu = pd.DataFrame({"A": [21, 22, 23, 24], "B": [30, 31, 32, 33]}) with pytest.raises(ValueError): calculate_averaged_congestion(congl, congu) def test_calculate_averaged_congestion_args_indices(): - congl = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [10, 11, 12, 13]}) - congu = pd.DataFrame({'C': [21, 22, 23, 24], 'D': [30, 31, 32, 33]}) + congl = pd.DataFrame({"A": [1, 2, 3, 4], "B": [10, 11, 12, 13]}) + congu = pd.DataFrame({"C": [21, 22, 23, 24], "D": [30, 31, 32, 33]}) with pytest.raises(ValueError): calculate_averaged_congestion(congl, congu) def test_calculate_averaged_congestion_returned_df_shape(): - congl = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [10, 11, 12, 13]}) - congu = pd.DataFrame({'A': [21, 22, 23, 24], 'B': [30, 31, 32, 33]}) + congl = pd.DataFrame({"A": [1, 2, 3, 4], "B": [10, 11, 12, 13]}) + congu = pd.DataFrame({"A": [21, 22, 23, 24], "B": [30, 31, 32, 33]}) assert calculate_averaged_congestion(congl, congu).shape == (2, 2) def test_calculate_averaged_congestion_returned_df_columns_name(): - congl = pd.DataFrame({'a': [1, 2, 3, 4], 'b': [10, 11, 12, 13]}) - congu = pd.DataFrame({'a': [21, 22, 23, 24], 'b': [30, 31, 32, 33]}) + congl = pd.DataFrame({"a": [1, 2, 3, 4], "b": [10, 11, 12, 13]}) + congu = pd.DataFrame({"a": [21, 22, 23, 24], "b": [30, 31, 32, 33]}) mean_cong = calculate_averaged_congestion(congl, congu) - assert np.array_equal(mean_cong.columns, ['CONGL', 'CONGU']) + assert np.array_equal(mean_cong.columns, ["CONGL", "CONGU"]) def test_calculate_averaged_congestion_returned_df_indices(): - congl = pd.DataFrame({'marge': [1, 2, 3, 4], 'homer': [10, 11, 12, 13]}) - congu = pd.DataFrame({'marge': [21, 22, 23, 24], 'homer': [30, 31, 32, 33]}) + congl = pd.DataFrame({"marge": [1, 2, 3, 4], "homer": [10, 11, 12, 13]}) + congu = pd.DataFrame({"marge": [21, 22, 23, 24], "homer": [30, 31, 32, 33]}) mean_cong = calculate_averaged_congestion(congl, congu) - assert np.array_equal(set(mean_cong.index), set(['marge', 'homer'])) + assert np.array_equal(set(mean_cong.index), set(["marge", "homer"])) def test_calculate_averaged_congestion_returned_df_values(): - congl = pd.DataFrame({'bart': [1, 2, 3, 4], 'lisa': [10, 11, 12, 13]}) - congu = pd.DataFrame({'bart': [21, 22, 23, 24], 'lisa': [30, 31, 32, 33]}) + congl = pd.DataFrame({"bart": [1, 2, 3, 4], "lisa": [10, 11, 12, 13]}) + congu = pd.DataFrame({"bart": [21, 22, 23, 24], "lisa": [30, 31, 32, 33]}) mean_cong = calculate_averaged_congestion(congl, congu) assert np.array_equal(mean_cong.values, [[2.5, 22.5], [11.5, 31.5]]) + + +def test_get_pkl_path(): + test_path = "/path/to/test/directory" + without_scenario = _get_pkl_path(test_path) + assert without_scenario("congu") == "/path/to/test/directory/CONGU.pkl" + + +def test_get_pkl_path_with_scenario(): + test_path = "/path/to/test/directory" + with_scenario = _get_pkl_path(test_path, "87") + assert with_scenario("congu") == "/path/to/test/directory/87_CONGU.pkl" + + +def test_cast_keys_as_lists(): + outputs = {"marge": 2, "homer": np.array([1, 2, 3, 4])} + expected_output = {"marge": [2], "homer": [1, 2, 3, 4]} + _cast_keys_as_lists(outputs) + assert outputs == expected_output + + +def test_result_num(): + result365 = "/path/to/test/result_365.mat" + assert result_num(result365) == 365 diff --git a/pyreisejl/utility/tests/test_helpers.py b/pyreisejl/utility/tests/test_helpers.py index 9ea4848d..6234a56c 100644 --- a/pyreisejl/utility/tests/test_helpers.py +++ b/pyreisejl/utility/tests/test_helpers.py @@ -1,7 +1,21 @@ +import glob +import os +import pathlib +import string +from io import StringIO + import numpy as np +import pandas as pd import pytest -from pyreisejl.utility.helpers import sec2hms +from pyreisejl.utility.helpers import ( + InvalidDateArgument, + extract_date_limits, + insert_in_file, + sec2hms, + validate_time_format, + validate_time_range, +) def test_sec2hms_arg_type(): @@ -34,3 +48,86 @@ def test_sec2hms_hours_only(): def test_sec2hms_hms(): seconds = 72 * 3600 + 45 * 60 + 15 assert sec2hms(seconds) == (72, 45, 15) + + +def test_validate_time_format_type(): + date = "2016/01/01" + with pytest.raises(InvalidDateArgument): + validate_time_format(date) + + +def test_validate_time_format_day(): + date = "2016-01-01" + assert validate_time_format(date) == pd.Timestamp("2016-01-01 00:00:00") + + +def test_validate_time_format_hours(): + date = "2016-01-01 12" + assert validate_time_format(date) == pd.Timestamp("2016-01-01 12:00:00") + + +def test_validate_time_format_min(): + date = "2016-01-01 12:30" + assert validate_time_format(date) == pd.Timestamp("2016-01-01 12:30:00") + + +def test_validate_time_format_sec(): + date = "2016-01-01 12:30:30" + assert validate_time_format(date) == pd.Timestamp("2016-01-01 12:30:30") + + +def test_validate_time_format_end_date(): + date = "2016-01-01" + assert validate_time_format(date, end_date=True) == pd.Timestamp( + "2016-01-01 23:00:00" + ) + + +def test_validate_time_range(): + date = "2020-06-01" + min_ts = "2016-01-01" + max_ts = "2016-12-31" + with pytest.raises(InvalidDateArgument): + validate_time_range(date, min_ts, max_ts) + + +def test_extract_date_limits(): + example_csv = StringIO() + example_csv.write( + """UTC, 301, 302, 303, 304, 305, 306, 307, 308 +1/1/16 0:00, 2965.292134, 1184.906904, 1676.760454, 4556.579997, 18295.21119, 8611.226983, 14600.71, 1830.2 +1/1/16 1:00, 3010.513011, 1215.345962, 1731.231093, 4684.33594, 18781.27034, 8959.356, 14655.5098, 1977.303 +1/1/16 2:00, 3002.107911, 1192.115238, 1709.06971, 4536.162029, 18296.94916, 8744.25244, 14236.594, 1849.04 +""" + ) + example_csv.seek(0) + + assert extract_date_limits(example_csv) == ( + pd.Timestamp("2016-01-01 00:00:00"), + pd.Timestamp("2016-01-01 02:00:00"), + "H", + ) + + +def test_insert_in_file(): + shape = (10, 100) + table = pd.DataFrame( + { + c: np.random.randint(0, 1000, size=shape[1]) + for c, _ in zip(list(string.ascii_lowercase)[: shape[0]], range(shape[0])) + } + ) + table.index.name = "id" + + filename = os.path.join( + pathlib.Path(__file__).parent.absolute(), "test_insert_in_file.csv" + ) + + table.to_csv(filename) + cell = (np.random.choice(table.index), np.random.choice(table.columns)) + try: + insert_in_file(filename, cell[0], cell[1], table.loc[cell]) + assert table.equals(pd.read_csv(filename, index_col=0)) + finally: + for f in glob.glob(filename + "*"): + os.remove(f) diff --git a/pyreisejl/utility/tests/test_state.py b/pyreisejl/utility/tests/test_state.py new file mode 100644 index 00000000..fd5addd9 --- /dev/null +++ b/pyreisejl/utility/tests/test_state.py @@ -0,0 +1,36 @@ +from subprocess import PIPE, Popen + +import pytest + +from pyreisejl.utility.state import ApplicationState, SimulationState + + +@pytest.fixture +def test_proc(): + cmd = ["echo", "foo"] + proc = Popen(cmd, stdout=PIPE, stderr=PIPE, start_new_session=True) + return proc + + +def test_scenario_state_refresh(test_proc): + entry = SimulationState(123, test_proc) + entry.as_dict() + assert entry.output == ["foo"] + assert entry.errors == [] + + +def test_scenario_state_serializable(test_proc): + entry = SimulationState(123, test_proc) + keys = entry.as_dict().keys() + assert "proc" not in keys + assert all(["listener" not in k for k in keys]) + + +def test_app_state_get(test_proc): + state = ApplicationState() + assert len(state.ongoing) == 0 + + entry = SimulationState(123, test_proc) + state.add(entry) + assert len(state.ongoing) == 1 + assert state.get(123) is not None diff --git a/requirements.txt b/requirements.txt index ead27c0c..68e286ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ -h5py==2.9.0 -ipython>=7.0 -julia==0.5.1 -numpy==1.16.0 -pandas==0.25.3 -scipy>=1.2 -tqdm==4.29.1 +h5py~=2.10.0 +ipython~=7.0 +julia~=0.5.1 +numpy~=1.18.5 +pandas~=1.1.3 +scipy~=1.2 +tqdm~=4.29.1 +pytest~=6.1.1 +Flask~=1.1.2 diff --git a/src/REISE.jl b/src/REISE.jl index 0d471705..7e9b6762 100644 --- a/src/REISE.jl +++ b/src/REISE.jl @@ -4,8 +4,8 @@ import CSV import DataFrames import Dates import JuMP -import Gurobi import MAT +import Requires import SparseArrays: sparse, SparseMatrixCSC @@ -19,6 +19,13 @@ include("query.jl") # Defines get_results (used in interval_loop) include("save.jl") # Defines save_input_mat, save_results +function __init__() + Requires.@require Gurobi="2e9cd046-0924-5485-92f1-d5272153d98b" begin + include(joinpath("solver_specific", "gurobi.jl")) + end +end + + """ REISE.run_scenario(; interval=24, n_interval=3, start_index=1, outputfolder="output", @@ -29,22 +36,29 @@ Run a scenario consisting of several intervals. 'n_interval' specifies the number of intervals in a scenario. 'start_index' specifies the starting hour of the first interval, to determine which time-series data should be loaded into each intervals. -'outputfolder' specifies where to store the results. This folder will be - created if it does not exist at runtime. 'inputfolder' specifies where to load the relevant data from. Required files are 'case.mat', 'demand.csv', 'hydro.csv', 'solar.csv', and 'wind.csv'. +'outputfolder' specifies where to store the results. Defaults to an `output` + subdirectory of inputfolder. This folder will be created if it does not exist at + runtime. +'optimizer_factory' is the solver used for optimization. If not specified, Gurobi is + used by default. """ function run_scenario(; num_segments::Int=1, interval::Int, n_interval::Int, start_index::Int, - inputfolder::String, outputfolder::Union{String, Nothing}=nothing) + inputfolder::String, outputfolder::Union{String, Nothing}=nothing, + threads::Union{Int, Nothing}=nothing, optimizer_factory=nothing, + solver_kwargs::Union{Dict, Nothing}=nothing) + isnothing(optimizer_factory) && error("optimizer_factory must be specified") # Setup things that build once + # If no solver kwargs passed, instantiate an empty dict + solver_kwargs = something(solver_kwargs, Dict()) # If outputfolder not given, by default assign it inside inputfolder isnothing(outputfolder) && (outputfolder = joinpath(inputfolder, "output")) # If outputfolder doesn't exist (isdir evaluates false) create it (mkdir) isdir(outputfolder) || mkdir(outputfolder) stdout_filepath = joinpath(outputfolder, "stdout.log") stderr_filepath = joinpath(outputfolder, "stderr.err") - env = Gurobi.Env() case = read_case(inputfolder) storage = read_storage(inputfolder) println("All scenario files loaded!") @@ -55,18 +69,17 @@ function run_scenario(; "storage" => storage, "interval_length" => interval, ) - solver_kwargs = Dict("Method" => 2, "Crossover" => 0) + # If a number of threads is specified, add to solver settings dict + isnothing(threads) || (solver_kwargs["Threads"] = threads) println("All preparation complete!") # While redirecting stdout and stderr... println("Redirecting outputs, see stdout.log & stderr.err in outputfolder") redirect_stdout_stderr(stdout_filepath, stderr_filepath) do # Loop through intervals - interval_loop(env, model_kwargs, solver_kwargs, interval, n_interval, - start_index, inputfolder, outputfolder) - GC.gc() - Gurobi.free_env(env) - println("Connection closed successfully!") + m = interval_loop(optimizer_factory, model_kwargs, solver_kwargs, interval, + n_interval, start_index, inputfolder, outputfolder) end + return m end # Module end diff --git a/src/loop.jl b/src/loop.jl index 822fe682..54465214 100644 --- a/src/loop.jl +++ b/src/loop.jl @@ -1,9 +1,21 @@ +"""Convert a dict with string keys to a NamedTuple, for python-eqsue kwargs splatting""" +function symbolize(d::Dict{String,Any})::NamedTuple + return (; (Symbol(k) => v for (k,v) in d)...) +end + + +function new_model(optimizer_factory)::JuMP.Model + return JuMP.Model(optimizer_factory) +end + + """ - interval_loop(env, model_kwargs, solver_kwargs, interval, n_interval, + interval_loop(factory_like, model_kwargs, solver_kwargs, interval, n_interval, start_index, inputfolder, outputfolder) Given: -- a Gurobi environment `env` +- optimizer instantiation object `factory_like`: + something that can be passed to new_model (goes to JuMP.Model by default) - a dictionary of model keyword arguments `model_kwargs` - a dictionary of solver keyword arguments `solver_kwargs` - an interval length `interval` (hours) @@ -15,7 +27,7 @@ Given: Build a model, and run through the intervals, re-building the model and/or re-setting constraint right-hand-side values as necessary. """ -function interval_loop(env::Gurobi.Env, model_kwargs::Dict, +function interval_loop(factory_like, model_kwargs::Dict, solver_kwargs::Dict, interval::Int, n_interval::Int, start_index::Int, inputfolder::String, outputfolder::String) @@ -45,10 +57,9 @@ function interval_loop(env::Gurobi.Env, model_kwargs::Dict, if storage_enabled model_kwargs["storage_e0"] = storage.sd_table.InitialStorage end - m_kwargs = (; (Symbol(k) => v for (k,v) in model_kwargs)...) - s_kwargs = (; (Symbol(k) => v for (k,v) in solver_kwargs)...) - m = JuMP.direct_model(Gurobi.Optimizer(env; s_kwargs...)) - m, voi = _build_model(m; m_kwargs...) + m = new_model(factory_like) + JuMP.set_optimizer_attributes(m, pairs(solver_kwargs)...) + m, voi = _build_model(m; symbolize(model_kwargs)...) elseif i == 2 # Build a model with an initial ramp constraint model_kwargs["initial_ramp_enabled"] = true @@ -56,10 +67,9 @@ function interval_loop(env::Gurobi.Env, model_kwargs::Dict, if storage_enabled model_kwargs["storage_e0"] = storage_e0 end - m_kwargs = (; (Symbol(k) => v for (k,v) in model_kwargs)...) - s_kwargs = (; (Symbol(k) => v for (k,v) in solver_kwargs)...) - m = JuMP.direct_model(Gurobi.Optimizer(env; s_kwargs...)) - m, voi = _build_model(m; m_kwargs...) + m = new_model(factory_like) + JuMP.set_optimizer_attributes(m, pairs(solver_kwargs)...) + m, voi = _build_model(m; symbolize(model_kwargs)...) else # Reassign right-hand-side of constraints to match profiles bus_demand = _make_bus_demand(case, interval_start, interval_end) @@ -118,34 +128,40 @@ function interval_loop(env::Gurobi.Env, model_kwargs::Dict, f = JuMP.objective_value(m) results = get_results(f, voi, model_kwargs["case"]) break + elseif ((status == JuMP.MOI.LOCALLY_SOLVED) + & ("load_shed_enabled" in keys(model_kwargs))) + # if load shedding is enabled, we'll accept 'suboptimal' + f = JuMP.objective_value(m) + results = get_results(f, voi, model_kwargs["case"]) + break elseif ((status in numeric_statuses) + & (JuMP.solver_name(m) == "Gurobi") & !("BarHomogeneous" in keys(solver_kwargs))) - # if BarHomogeneous is not enabled, enable it and re-build + # if Gurobi, and BarHomogeneous is not enabled, enable it and re-solve solver_kwargs["BarHomogeneous"] = 1 println("enable BarHomogeneous") - JuMP.set_parameter(m, "BarHomogeneous", 1) + JuMP.set_optimizer_attribute(m, "BarHomogeneous", 1) elseif ((status in infeasible_statuses) & !("load_shed_enabled" in keys(model_kwargs))) # if load shed not enabled, enable it and re-build the model model_kwargs["load_shed_enabled"] = true - m_kwargs = (; (Symbol(k) => v for (k,v) in model_kwargs)...) - s_kwargs = (; (Symbol(k) => v for (k,v) in solver_kwargs)...) println("rebuild with load shed") - m = JuMP.direct_model(Gurobi.Optimizer(env; s_kwargs...)) - m, voi = _build_model(m; m_kwargs...) + m = new_model(factory_like) + JuMP.set_optimizer_attributes(m, pairs(solver_kwargs)...) + m, voi = _build_model(m; symbolize(model_kwargs)...) intervals_without_loadshed = 0 - elseif !("BarHomogeneous" in keys(solver_kwargs)) - # if BarHomogeneous is not enabled, enable it and re-build + elseif ((JuMP.solver_name(m) == "Gurobi") + & !("BarHomogeneous" in keys(solver_kwargs))) + # if Gurobi, and BarHomogeneous is not enabled, enable it and re-solve solver_kwargs["BarHomogeneous"] = 1 println("enable BarHomogeneous") - JuMP.set_parameter(m, "BarHomogeneous", 1) + JuMP.set_optimizer_attribute(m, "BarHomogeneous", 1) elseif !("load_shed_enabled" in keys(model_kwargs)) model_kwargs["load_shed_enabled"] = true - m_kwargs = (; (Symbol(k) => v for (k,v) in model_kwargs)...) - s_kwargs = (; (Symbol(k) => v for (k,v) in solver_kwargs)...) println("rebuild with load shed") - m = JuMP.direct_model(Gurobi.Optimizer(env; s_kwargs...)) - m, voi = _build_model(m; m_kwargs...) + m = new_model(factory_like) + JuMP.set_optimizer_attributes(m, pairs(solver_kwargs)...) + m, voi = _build_model(m; symbolize(model_kwargs)...) intervals_without_loadshed = 0 else # Something has gone very wrong @@ -167,13 +183,13 @@ function interval_loop(env::Gurobi.Env, model_kwargs::Dict, error("Unknown status code!") end end - + # Save initial conditions for next interval pg0 = results.pg[:,end] if storage_enabled storage_e0 = results.storage_e[:,end] end - + # Save results results_filename = "result_" * string(i-1) * ".mat" results_filepath = joinpath(outputfolder, results_filename) @@ -193,11 +209,12 @@ function interval_loop(env::Gurobi.Env, model_kwargs::Dict, # delete! will work here even if the key is not present delete!(solver_kwargs, "BarHomogeneous") delete!(model_kwargs, "load_shed_enabled") - m_kwargs = (; (Symbol(k) => v for (k,v) in model_kwargs)...) - s_kwargs = (; (Symbol(k) => v for (k,v) in solver_kwargs)...) - m = JuMP.direct_model(Gurobi.Optimizer(env; s_kwargs...)) - m, voi = _build_model(m; m_kwargs...) + m = new_model(factory_like) + JuMP.set_optimizer_attributes(m, pairs(solver_kwargs)...) + m, voi = _build_model(m; symbolize(model_kwargs)...) end end end + + return m end diff --git a/src/model.jl b/src/model.jl index 01a3d146..af53ef0a 100644 --- a/src/model.jl +++ b/src/model.jl @@ -46,12 +46,13 @@ function _make_bus_demand(case::Case, start_index::Int, end_index::Int)::Matrix bus_idx = 1:length(case.busid) bus_df = DataFrames.DataFrame( name=case.busid, load=case.bus_demand, zone=case.bus_zone) - zone_demand = DataFrames.by(bus_df, :zone, :load => sum) + zone_demand = DataFrames.combine( + DataFrames.groupby(bus_df, :zone), :load => sum) zone_list = sort(collect(Set(case.bus_zone))) num_zones = length(zone_list) zone_idx = 1:num_zones zone_id2idx = Dict(zone_list .=> zone_idx) - bus_df_with_zone_load = join(bus_df, zone_demand, on = :zone) + bus_df_with_zone_load = DataFrames.innerjoin(bus_df, zone_demand, on=:zone) bus_share = bus_df[:, :load] ./ bus_df_with_zone_load[:, :load_sum] bus_zone_idx = Int64[zone_id2idx[z] for z in case.bus_zone] zone_to_bus_shares = sparse( @@ -93,15 +94,15 @@ function _make_sets(case::Case, storage::Union{Storage,Nothing})::Sets # Positional indices from mpc.gencost MODEL = 1 NCOST = 4 - # Buses + # Sets - Buses num_bus = length(case.busid) bus_idx = 1:num_bus bus_id2idx = Dict(case.busid .=> bus_idx) load_bus_idx = findall(case.bus_demand .> 0) num_load_bus = length(load_bus_idx) # Sets - branches - branch_rating = vcat(case.branch_rating, case.dcline_rating) - branch_rating[branch_rating .== 0] .= Inf + ac_branch_rating = replace(case.branch_rating, 0=>Inf) + branch_rating = vcat(ac_branch_rating, case.dcline_pmax) num_branch_ac = length(case.branchid) num_branch = num_branch_ac + length(case.dclineid) branch_idx = 1:num_branch @@ -195,7 +196,8 @@ function _build_model(m::JuMP.Model; case::Case, storage::Storage, segment_slope = _build_segment_slope(case, sets.segment_idx, segment_width) # Branch connectivity matrix branch_map = _make_branch_map(case) - branch_rating = vcat(case.branch_rating, case.dcline_rating) + branch_pmin = vcat(-1 * case.branch_rating, case.dcline_pmin) + branch_pmax = vcat(case.branch_rating, case.dcline_pmax) # Demand by bus bus_demand = _make_bus_demand(case, start_index, end_index) bus_demand *= demand_scaling @@ -225,7 +227,7 @@ function _build_model(m::JuMP.Model; case::Case, storage::Storage, end) if load_shed_enabled JuMP.@variable(m, - 0 <= load_shed[i in 1:sets.num_load_bus, j in hour_idx] + 0 <= load_shed[i in 1:sets.num_load_bus, j in 1:interval_length] <= bus_demand[sets.load_bus_idx[i], j], container=Array) end @@ -271,20 +273,30 @@ function _build_model(m::JuMP.Model; case::Case, storage::Storage, if storage_enabled println("storage soc_tracking: ", Dates.now()) JuMP.@constraint(m, - soc_tracking[i in sets.storage_idx, h in 1:(num_hour-1)], + soc_tracking[i in 1:sets.num_storage, h in 1:(num_hour-1)], storage_soc[i, h+1] == ( - storage_soc[i, h] + storage_soc[i, h] * (1 - storage.sd_table.LossFactor[i]) + storage.sd_table.InEff[i] * storage_chg[i, h+1] - (1 / storage.sd_table.OutEff[i]) * storage_dis[i, h+1]), container=Array) println("storage initial_soc: ", Dates.now()) JuMP.@constraint(m, - initial_soc[i in sets.storage_idx], + initial_soc[i in 1:sets.num_storage], storage_soc[i, 1] == ( storage_e0[i] + storage.sd_table.InEff[i] * storage_chg[i, 1] - (1 / storage.sd_table.OutEff[i]) * storage_dis[i, 1]), container=Array) + println("storage final_soc_min: ", Dates.now()) + JuMP.@constraint(m, + soc_terminal_min[i in 1:sets.num_storage], + storage_soc[i, num_hour] >= storage.sd_table.ExpectedTerminalStorageMin[i], + container=Array) + println("storage final_soc_max: ", Dates.now()) + JuMP.@constraint(m, + soc_terminal_max[i in 1:sets.num_storage], + storage_soc[i, num_hour] <= storage.sd_table.ExpectedTerminalStorageMax[i], + container=Array) end noninf_ramp_idx = findall(case.gen_ramp30 .!= Inf) @@ -320,21 +332,24 @@ function _build_model(m::JuMP.Model; case::Case, storage::Storage, pg[i, h] == case.gen_pmin[i] + sum(pg_seg[i, :, h])) if trans_viol_enabled - JuMP.@expression(m, - branch_limit, branch_rating + trans_viol) + JuMP.@expression(m, branch_limit_pmin, branch_pmin - trans_viol) + JuMP.@expression(m, branch_limit_pmax, branch_pmax + trans_viol) else JuMP.@expression(m, - branch_limit[br in sets.branch_idx, h in hour_idx], - branch_rating[br]) + branch_limit_pmin[br in sets.branch_idx, h in hour_idx], + branch_pmin[br]) + JuMP.@expression(m, + branch_limit_pmax[br in sets.branch_idx, h in hour_idx], + branch_pmax[br]) end println("branch_min, branch_max: ", Dates.now()) JuMP.@constraint(m, branch_min[br in sets.noninf_branch_idx, h in hour_idx], - -1 * branch_limit[br, h] <= pf[br, h]) + branch_limit_pmin[br, h] <= pf[br, h]) println("branch_max: ", Dates.now()) JuMP.@constraint(m, branch_max[br in sets.noninf_branch_idx, h in hour_idx], - pf[br, h] <= branch_limit[br, h]) + pf[br, h] <= branch_limit_pmax[br, h]) println("branch_angle: ", Dates.now()) # Explicit numbering here so that we constrain AC branches but not DC diff --git a/src/read.jl b/src/read.jl index 2bc277d0..dd13658c 100644 --- a/src/read.jl +++ b/src/read.jl @@ -27,12 +27,14 @@ function read_case(filepath) end case["dcline_from"] = convert(Array{Int,1}, mpc["dcline"][:,1]) case["dcline_to"] = convert(Array{Int,1}, mpc["dcline"][:,2]) - case["dcline_rating"] = mpc["dcline"][:,11] + case["dcline_pmin"] = mpc["dcline"][:,10] + case["dcline_pmax"] = mpc["dcline"][:,11] else case["dclineid"] = Int64[] case["dcline_from"] = Int64[] case["dcline_to"] = Int64[] - case["dcline_rating"] = Float64[] + case["dcline_pmin"] = Float64[] + case["dcline_pmax"] = Float64[] end # Buses diff --git a/src/save.jl b/src/save.jl index 1b8ad862..502582e6 100644 --- a/src/save.jl +++ b/src/save.jl @@ -31,8 +31,8 @@ function save_input_mat(case::Case, storage::Storage, inputfolder::String, num_storage = size(storage.gen, 1) # Add fuel types for storage 'generators'. ESS = energy storage system mpc["genfuel"] = [mpc["genfuel"] ; repeat(["ess"], num_storage)] - # Save storage modifications to gen table (with extra zeros for MUs) - mpc["gen"] = [mpc["gen"] ; hcat(storage.gen, zeros(num_storage, 4))] + # Save storage modifications to gen table + mpc["gen"] = [mpc["gen"] ; storage.gen] # Save storage modifications to gencost #@show case.gencost[:, gencost_MODEL] segment_indices = findall(case.gencost[:, gencost_MODEL] .== 1) diff --git a/src/solver_specific/gurobi.jl b/src/solver_specific/gurobi.jl new file mode 100644 index 00000000..0f29fe7e --- /dev/null +++ b/src/solver_specific/gurobi.jl @@ -0,0 +1,23 @@ +# Importing Gurobi in this way avoids a warning with Requires +import .Gurobi + + +function run_scenario_gurobi(; solver_kwargs::Union{Dict, Nothing}=nothing, kwargs...) + solver_kwargs = something(solver_kwargs, Dict("Method" => 2, "Crossover" => 0)) + try + global env = Gurobi.Env() + global m = run_scenario(; + optimizer_factory=env, solver_kwargs=solver_kwargs, kwargs...) + finally + Gurobi.finalize(JuMP.backend(m)) + Gurobi.finalize(env) + println("Connection closed successfully!") + end + # Return `nothing` to prevent `m` from the `try` block from being returned + return nothing +end + + +function new_model(env::Gurobi.Env) + return JuMP.direct_model(Gurobi.Optimizer(env)) +end diff --git a/src/types.jl b/src/types.jl index 34239ee3..82dab31d 100644 --- a/src/types.jl +++ b/src/types.jl @@ -11,7 +11,8 @@ Base.@kwdef struct Case dclineid::Array{Int64,1} dcline_from::Array{Int64,1} dcline_to::Array{Int64,1} - dcline_rating::Array{Float64,1} + dcline_pmin::Array{Float64,1} + dcline_pmax::Array{Float64,1} busid::Array{Int64,1} bus_demand::Array{Float64,1} diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..d11b439b --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +envlist = pytest, format, flake8 +skipsdist = true + +[testenv] +deps = + pytest: -rrequirements.txt + {format,checkformatting}: black + {format,checkformatting}: isort + flake8: flake8 +changedir = pyreisejl +commands = + pytest: pytest + format: black . + format: isort . + checkformatting: black . --check --diff + checkformatting: isort --check --diff . + flake8: flake8 + +[flake8] +ignore = E501,E731 + +[isort] +profile = black