diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 226c1de..34ab1cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,6 +8,7 @@ on: jobs: CodeQL: + name: CodeQL validation runs-on: ubuntu-latest permissions: security-events: write @@ -23,6 +24,28 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 + OpenAPI: + runs-on: ubuntu-latest + name: OpenAPI validation + + # Service containers to run with `runner-job` + services: + # Label used to access the service container + swagger-editor: + # Docker Hub image + image: swaggerapi/swagger-editor + ports: + # Maps port 8080 on service container to the host 80 + - 80:8080 + + steps: + - uses: actions/checkout@v2 + - name: Validate OpenAPI definition + uses: swaggerexpert/swagger-editor-validate@452076dc45d5d1f09dd55440c9bffc372de4da25 # Jul 29, 2024 + with: + swagger-editor-url: http://localhost/ + definition-file: docs/openapi.yaml + Build: runs-on: ubuntu-latest steps: @@ -46,7 +69,7 @@ jobs: - name: Test run: | - go install github.com/jstemmer/go-junit-report/v2@latest + go install github.com/jstemmer/go-junit-report/v2@85bf471 # Oct 18, 2023 name="$(ls aquarium-fish-*.linux_amd64)" FISH_PATH="$PWD/$name" go test -v -failfast -parallel 4 -count=1 ./tests/... 2>&1 | go-junit-report -iocopy -set-exit-code -out report.xml diff --git a/build.sh b/build.sh index dff0109..caa78a7 100755 --- a/build.sh +++ b/build.sh @@ -23,10 +23,10 @@ export CGO_ENABLED=0 echo "--- GENERATE CODE FOR AQUARIUM-FISH ---" # Install oapi-codegen if it's not available or version is not the same with go.mod gopath=$(go env GOPATH) -req_ver=$(grep -F 'github.com/deepmap/oapi-codegen' go.mod | cut -d' ' -f 2) +req_ver=$(grep -F 'github.com/oapi-codegen/oapi-codegen/v2' go.mod | cut -d' ' -f 2) curr_ver="$(PATH="$gopath/bin:$PATH" oapi-codegen --version 2>/dev/null | tail -1 || true)" if [ "$curr_ver" != "$req_ver" ]; then - go install "github.com/deepmap/oapi-codegen/cmd/oapi-codegen@$req_ver" + go install "github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@$req_ver" fi # Cleanup the old generated files find ./lib -name '*.gen.go' -delete diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 13405fc..bbc7b24 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -564,6 +564,35 @@ paths: security: - basic_auth: [] + /api/v1/task/{task_uid}: + get: + summary: Get ApplicationTask data + description: Returns the Application Task + operationId: ApplicationTaskGet + tags: + - Application + parameters: + - name: task_uid + in: path + description: UID of the Task + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/ApplicationTask' + '401': + $ref: '#/components/responses/UnauthorizedError' + '404': + description: ApplicationTask not found + security: + - basic_auth: [] + /api/v1/application/{uid}/deallocate: get: summary: Triggers Application deallocate @@ -745,7 +774,7 @@ paths: - name: handler in: path description: Which pprof handler to use. If empty - will show index - required: false + required: true schema: type: string responses: diff --git a/go.mod b/go.mod index 415bc65..152ba2b 100644 --- a/go.mod +++ b/go.mod @@ -10,22 +10,23 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.32.3 github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.10 github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 - github.com/deepmap/oapi-codegen v1.12.4 - github.com/getkin/kin-openapi v0.115.0 + github.com/getkin/kin-openapi v0.124.0 github.com/ghodss/yaml v1.0.0 github.com/glebarez/sqlite v1.7.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.5.0 github.com/gorilla/websocket v1.4.0 github.com/hpcloud/tail v1.0.0 - github.com/labstack/echo/v4 v4.10.2 + github.com/labstack/echo/v4 v4.11.4 github.com/mostlygeek/arp v0.0.0-20170424181311-541a2129847a + github.com/oapi-codegen/oapi-codegen/v2 v2.3.0 + github.com/oapi-codegen/runtime v1.1.1 github.com/rqlite/sql v0.0.0-20221103124402-8f9ff0ceb8f0 github.com/shirou/gopsutil/v3 v3.23.1 github.com/spf13/cobra v1.7.0 github.com/steinfletcher/apitest v1.5.15 github.com/ulikunitz/xz v0.5.11 - golang.org/x/crypto v0.8.0 - golang.org/x/net v0.9.0 + golang.org/x/crypto v0.17.0 + golang.org/x/net v0.19.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/gorm v1.24.6 ) @@ -42,22 +43,22 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/glebarez/go-sqlite v1.20.3 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/swag v0.21.1 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/swag v0.22.8 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/yaml v0.1.0 // indirect + github.com/invopop/yaml v0.2.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/labstack/gommon v0.4.0 // indirect + github.com/labstack/gommon v0.4.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/perimeterx/marshmallow v1.1.4 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -66,9 +67,9 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/time v0.5.0 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7a0674f..21520a1 100644 --- a/go.sum +++ b/go.sum @@ -27,18 +27,15 @@ github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deepmap/oapi-codegen v1.12.4 h1:pPmn6qI9MuOtCz82WY2Xaw46EQjgvxednXXrP7g5Q2s= -github.com/deepmap/oapi-codegen v1.12.4/go.mod h1:3lgHGMu6myQ2vqbbTXH2H1o4eXFTGnFiDaOaKKl5yas= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/getkin/kin-openapi v0.115.0 h1:c8WHRLVY3G8m9jQTy0/DnIuljgRwTCB5twZytQS4JyU= -github.com/getkin/kin-openapi v0.115.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= +github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= +github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= @@ -47,11 +44,10 @@ github.com/glebarez/sqlite v1.7.0 h1:A7Xj/KN2Lvie4Z4rrgQHY8MsbebX3NyWsL3n2i82MVI github.com/glebarez/sqlite v1.7.0/go.mod h1:PkeevrRlF/1BhQBCnzcMWzgrIk7IOop+qS2jUYLfHhk= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= +github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= @@ -61,17 +57,16 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= -github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= +github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -83,37 +78,33 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= -github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= -github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= -github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= +github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8= +github.com/labstack/echo/v4 v4.11.4/go.mod h1:noh7EvLwqDsmh/X/HWKPUl1AjzJrhyptRyEbQJfxen8= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mostlygeek/arp v0.0.0-20170424181311-541a2129847a h1:AfneHvfmYgUIcgdUrrDFklLdEzQAvG9AKRTe1x1mx/0= github.com/mostlygeek/arp v0.0.0-20170424181311-541a2129847a/go.mod h1:jZxafo9CAqaKFQE4zitrg5QNlA6CXUsjwXPlIppF3tk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw= -github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/oapi-codegen/oapi-codegen/v2 v2.3.0 h1:rICjNsHbPP1LttefanBPnwsSwl09SqhCO7Ee623qR84= +github.com/oapi-codegen/oapi-codegen/v2 v2.3.0/go.mod h1:4k+cJeSq5ntkwlcpQSxLxICCxQzCL772o30PxdibRt4= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= @@ -121,6 +112,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M= github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rqlite/sql v0.0.0-20221103124402-8f9ff0ceb8f0 h1:C8DZB5okjhCSd7zvkOM+zxGz7S6ulUFIL34bpkqFk+0= github.com/rqlite/sql v0.0.0-20221103124402-8f9ff0ceb8f0/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -137,66 +130,56 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= -github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/drivers/aws/driver.go b/lib/drivers/aws/driver.go index 42477cc..670c363 100644 --- a/lib/drivers/aws/driver.go +++ b/lib/drivers/aws/driver.go @@ -83,6 +83,7 @@ func (d *Driver) Prepare(config []byte) error { // Fill up the available tasks to execute d.tasks_list = append(d.tasks_list, &TaskSnapshot{driver: d}, + &TaskImage{driver: d}, ) d.quotas_mutex.Lock() diff --git a/lib/drivers/aws/options.go b/lib/drivers/aws/options.go index 234fc89..49bc773 100644 --- a/lib/drivers/aws/options.go +++ b/lib/drivers/aws/options.go @@ -29,7 +29,7 @@ import ( * somekey: somevalue */ type Options struct { - Image string `json:"image"` // ID/Name of the image to use + Image string `json:"image"` // ID/Name of the image you want to use (name that contains * is usually a bad idea for reproducibility) InstanceType string `json:"instance_type"` // Type of the instance from aws available list SecurityGroup string `json:"security_group"` // ID/Name of the security group to use for the instance Tags map[string]string `json:"tags"` // Tags to add during instance creation @@ -38,6 +38,10 @@ type Options struct { UserDataFormat string `json:"userdata_format"` // If not empty - will store the resource metadata to userdata in defined format UserDataPrefix string `json:"userdata_prefix"` // Optional if need to add custom prefix to the metadata key during formatting + + // TaskImage options + TaskImageName string `json:"task_image_name"` // Create new image with defined name + "-DATE.TIME" suffix + TaskImageEncryptKey string `json:"task_image_encrypt_key"` // KMS Key ID or Alias in format "alias/" if need to re-encrypt the newly created AMI snapshots } func (o *Options) Apply(options util.UnparsedJson) error { diff --git a/lib/drivers/aws/task_image.go b/lib/drivers/aws/task_image.go new file mode 100644 index 0000000..0dd82b1 --- /dev/null +++ b/lib/drivers/aws/task_image.go @@ -0,0 +1,279 @@ +/** + * Copyright 2024 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +package aws + +import ( + "context" + "encoding/json" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/adobe/aquarium-fish/lib/drivers" + "github.com/adobe/aquarium-fish/lib/log" + "github.com/adobe/aquarium-fish/lib/openapi/types" +) + +type TaskImage struct { + driver *Driver `json:"-"` + + *types.ApplicationTask `json:"-"` // Info about the requested task + *types.LabelDefinition `json:"-"` // Info about the used label definition + *types.Resource `json:"-"` // Info about the processed resource + + Full bool `json:"full"` // Make full (all disks including connected disks), or just the root OS disk image +} + +func (t *TaskImage) Name() string { + return "image" +} + +func (t *TaskImage) Clone() drivers.ResourceDriverTask { + n := *t + return &n +} + +func (t *TaskImage) SetInfo(task *types.ApplicationTask, def *types.LabelDefinition, res *types.Resource) { + t.ApplicationTask = task + t.LabelDefinition = def + t.Resource = res +} + +// Image could be executed during ALLOCATED & DEALLOCATE ApplicationStatus +func (t *TaskImage) Execute() (result []byte, err error) { + if t.ApplicationTask == nil { + return []byte(`{"error":"internal: invalid application task"}`), log.Error("AWS: Invalid application task:", t.ApplicationTask) + } + if t.LabelDefinition == nil { + return []byte(`{"error":"internal: invalid label definition"}`), log.Error("TEST: Invalid label definition:", t.LabelDefinition) + } + if t.Resource == nil || t.Resource.Identifier == "" { + return []byte(`{"error":"internal: invalid resource"}`), log.Error("AWS: Invalid resource:", t.Resource) + } + log.Infof("AWS: TaskImage %s: Creating image for Application %s", t.ApplicationTask.UID, t.ApplicationTask.ApplicationUID) + conn := t.driver.newEC2Conn() + + var opts Options + if err := opts.Apply(t.LabelDefinition.Options); err != nil { + log.Error("AWS: Unable to apply options:", err) + return []byte(`{"error":"internal: unable to apply label definition options"}`), log.Errorf("AWS: Unable to apply label definition options: %w", err) + } + + if t.ApplicationTask.When == types.ApplicationStatusDEALLOCATE { + // We need to stop the instance before creating image to ensure it will be consistent + input := ec2.StopInstancesInput{ + InstanceIds: []string{t.Resource.Identifier}, + } + + log.Infof("AWS: TaskImage %s: Stopping instance %q", t.ApplicationTask.UID, t.Resource.Identifier) + result, err := conn.StopInstances(context.TODO(), &input) + if err != nil { + // Do not fail hard here - it's still possible to take image of the instance + log.Error("AWS: Error during stopping the instance:", t.Resource.Identifier, err) + } + if len(result.StoppingInstances) < 1 || *result.StoppingInstances[0].InstanceId != t.Resource.Identifier { + // Do not fail hard here - it's still possible to take image of the instance + log.Error("AWS: Wrong instance id result during stopping:", t.Resource.Identifier) + } + } + + log.Debugf("AWS: TaskImage %s: Detecting block devices of the instance...", t.ApplicationTask.UID) + var block_devices []ec2_types.BlockDeviceMapping + + // In case we need just the root disk (!Full) - let's get some additional data + // We don't need to fill the block devices if we want a full image of the instance + if !t.Full { + // TODO: Probably better to use DescribeInstances + // Look for the root device name of the instance + describe_input := ec2.DescribeInstanceAttributeInput{ + InstanceId: aws.String(t.Resource.Identifier), + Attribute: ec2_types.InstanceAttributeNameRootDeviceName, + } + describe_resp, err := conn.DescribeInstanceAttribute(context.TODO(), &describe_input) + if err != nil { + return []byte{}, log.Errorf("AWS: Unable to request the instance RootDeviceName attribute %s: %v", t.Resource.Identifier, err) + } + root_device := aws.ToString(describe_resp.RootDeviceName.Value) + + // Looking for the instance block device mappings to clarify what we need to include in the image + describe_input = ec2.DescribeInstanceAttributeInput{ + InstanceId: aws.String(t.Resource.Identifier), + Attribute: ec2_types.InstanceAttributeNameBlockDeviceMapping, + } + describe_resp, err = conn.DescribeInstanceAttribute(context.TODO(), &describe_input) + if err != nil { + return []byte{}, log.Errorf("AWS: Unable to request the instance BlockDeviceMapping attribute %s: %v", t.Resource.Identifier, err) + } + + // Filter the block devices in the image if we don't need full one + for _, dev := range describe_resp.BlockDeviceMappings { + // Requesting volume to get necessary data for required Ebs field + mapping := ec2_types.BlockDeviceMapping{ + DeviceName: dev.DeviceName, + } + if root_device != aws.ToString(dev.DeviceName) { + mapping.NoDevice = aws.String("") + } else { + log.Debugf("AWS: TaskImage %s: Only root disk will be used to create image: %s", t.ApplicationTask.UID, root_device) + if dev.Ebs == nil { + return []byte{}, log.Errorf("AWS: Root disk doesn't have EBS configuration") + } + params := ec2.DescribeVolumesInput{ + VolumeIds: []string{aws.ToString(dev.Ebs.VolumeId)}, + } + vol_resp, err := conn.DescribeVolumes(context.TODO(), ¶ms) + if err != nil || len(vol_resp.Volumes) < 1 { + return []byte{}, log.Errorf("AWS: Unable to request the instance root volume info %s: %v", aws.ToString(dev.Ebs.VolumeId), err) + } + vol_info := vol_resp.Volumes[0] + mapping.Ebs = &ec2_types.EbsBlockDevice{ + DeleteOnTermination: dev.Ebs.DeleteOnTermination, + //Encrypted: vol_info.Encrypted, + //Iops: vol_info.Iops, + //KmsKeyId: vol_info.KmsKeyId, + //OutpostArn: vol_info.OutpostArn, + //SnapshotId: vol_info.SnapshotId, + //Throughput: vol_info.Throughput, + VolumeSize: vol_info.Size, + VolumeType: vol_info.VolumeType, + } + } + block_devices = append(block_devices, mapping) + } + } else { + log.Debugf("AWS: TaskImage %s: All the instance disks will be used for image", t.ApplicationTask.UID) + } + + // Preparing the create image request + image_name := opts.Image + time.Now().UTC().Format("-060102.150405") + if opts.TaskImageName != "" { + image_name = opts.TaskImageName + time.Now().UTC().Format("-060102.150405") + } + input := ec2.CreateImageInput{ + InstanceId: aws.String(t.Resource.Identifier), + Name: aws.String(image_name), + BlockDeviceMappings: block_devices, + Description: aws.String("Created by AquariumFish"), + NoReboot: aws.Bool(true), // Action wants to do that on running instance or already stopped one + TagSpecifications: []ec2_types.TagSpecification{{ + ResourceType: ec2_types.ResourceTypeImage, + Tags: []ec2_types.Tag{ + { + Key: aws.String("InstanceId"), + Value: aws.String(t.Resource.Identifier), + }, + { + Key: aws.String("ApplicationTask"), + Value: aws.String(t.ApplicationTask.UID.String()), + }, + { + Key: aws.String("ParentImage"), + Value: aws.String(opts.Image), + }, + }, + }}, + } + if opts.TaskImageEncryptKey != "" { + // Append tmp to the name since it's just a temporary image for further re-encryption + input.Name = aws.String("tmp_" + image_name) + } + + if t.ApplicationTask.When == types.ApplicationStatusDEALLOCATE { + // Wait for instance stopped before going forward with image creation + log.Infof("AWS: TaskImage %s: Wait for instance %q stopping...", t.ApplicationTask.UID, t.Resource.Identifier) + sw := ec2.NewInstanceStoppedWaiter(conn) + max_wait := 10 * time.Minute + wait_input := ec2.DescribeInstancesInput{ + InstanceIds: []string{ + t.Resource.Identifier, + }, + } + if err := sw.Wait(context.TODO(), &wait_input, max_wait); err != nil { + // Do not fail hard here - it's still possible to create image of the instance + log.Error("AWS: Error during wait for instance stop:", t.Resource.Identifier, err) + } + } + log.Debugf("AWS: TaskImage %s: Creating image with name %q...", t.ApplicationTask.UID, aws.ToString(input.Name)) + resp, err := conn.CreateImage(context.TODO(), &input) + if err != nil { + return []byte{}, log.Errorf("AWS: Unable to create image from instance %s: %v", t.Resource.Identifier, err) + } + if resp.ImageId == nil { + return []byte{}, log.Errorf("AWS: No image was created from instance %s", t.Resource.Identifier) + } + + image_id := aws.ToString(resp.ImageId) + log.Infof("AWS: TaskImage %s: Created image %q with id %q...", t.ApplicationTask.UID, aws.ToString(input.Name), image_id) + + // If TaskImageEncryptKey is set - we need to copy the image with enabled encryption and delete the temp one + if opts.TaskImageEncryptKey != "" { + // Wait for the tmp image to be completed, otherwise if we will start a copy - it will fail... + log.Infof("AWS: TaskImage %s: Wait for image %s %q availability...", t.ApplicationTask.UID, image_id, aws.ToString(input.Name)) + sw := ec2.NewImageAvailableWaiter(conn) + max_wait := 60 * time.Minute + wait_input := ec2.DescribeImagesInput{ + ImageIds: []string{ + image_id, + }, + } + if err := sw.Wait(context.TODO(), &wait_input, max_wait); err != nil { + // Do not fail hard here - we still need to remove the tmp image + log.Error("AWS: Error during wait for image availability:", image_id, err) + } else { + copy_input := ec2.CopyImageInput{ + Name: aws.String(image_name), + Description: input.Description, + SourceImageId: resp.ImageId, + SourceRegion: aws.String(t.driver.cfg.Region), + CopyImageTags: aws.Bool(true), + Encrypted: aws.Bool(true), + KmsKeyId: aws.String(opts.TaskImageEncryptKey), + } + log.Infof("AWS: TaskImage %s: Re-encrypting tmp image to final image %q", t.ApplicationTask.UID, aws.ToString(copy_input.Name)) + resp, err := conn.CopyImage(context.TODO(), ©_input) + if err != nil { + return []byte{}, log.Errorf("AWS: Unable to copy image from tmp image %s: %v", aws.ToString(resp.ImageId), err) + } + if resp.ImageId == nil { + return []byte{}, log.Errorf("AWS: No image was copied from tmp image %s", aws.ToString(resp.ImageId)) + } + // Wait for the image to be completed, otherwise if we will delete the temp one right away it will fail... + log.Infof("AWS: TaskImage %s: Wait for re-encrypted image %s %q availability...", t.ApplicationTask.UID, aws.ToString(resp.ImageId), image_name) + sw := ec2.NewImageAvailableWaiter(conn) + max_wait := 60 * time.Minute + wait_input := ec2.DescribeImagesInput{ + ImageIds: []string{ + aws.ToString(resp.ImageId), + }, + } + if err := sw.Wait(context.TODO(), &wait_input, max_wait); err != nil { + // Do not fail hard here - we still need to remove the tmp image + log.Error("AWS: Error during wait for re-encrypted image availability:", aws.ToString(resp.ImageId), err) + } + } + + // Delete the temp image & associated snapshots + log.Debugf("AWS: TaskImage %s: Deleting the temp image %q", t.ApplicationTask.UID, image_id) + if err = t.driver.deleteImage(conn, image_id); err != nil { + return []byte{}, log.Errorf("AWS: Unable to create image from instance %s: %v", t.Resource.Identifier, err) + } + + image_id = aws.ToString(resp.ImageId) + } + + log.Infof("AWS: Created image for the instance %s: %s %q", t.Resource.Identifier, image_id, image_name) + + return json.Marshal(map[string]string{"image": image_id, "image_name": image_name}) +} diff --git a/lib/drivers/aws/tasks.go b/lib/drivers/aws/task_snapshot.go similarity index 72% rename from lib/drivers/aws/tasks.go rename to lib/drivers/aws/task_snapshot.go index a8910c3..3e175ac 100644 --- a/lib/drivers/aws/tasks.go +++ b/lib/drivers/aws/task_snapshot.go @@ -31,6 +31,7 @@ type TaskSnapshot struct { driver *Driver `json:"-"` *types.ApplicationTask `json:"-"` // Info about the requested task + *types.LabelDefinition `json:"-"` // Info about the used label definition *types.Resource `json:"-"` // Info about the processed resource Full bool `json:"full"` // Make full (all disks including OS image), or just the additional disks snapshot @@ -45,8 +46,9 @@ func (t *TaskSnapshot) Clone() drivers.ResourceDriverTask { return &n } -func (t *TaskSnapshot) SetInfo(task *types.ApplicationTask, res *types.Resource) { +func (t *TaskSnapshot) SetInfo(task *types.ApplicationTask, def *types.LabelDefinition, res *types.Resource) { t.ApplicationTask = task + t.LabelDefinition = def t.Resource = res } @@ -55,18 +57,23 @@ func (t *TaskSnapshot) Execute() (result []byte, err error) { if t.ApplicationTask == nil { return []byte(`{"error":"internal: invalid application task"}`), log.Error("AWS: Invalid application task:", t.ApplicationTask) } + if t.LabelDefinition == nil { + return []byte(`{"error":"internal: invalid label definition"}`), log.Error("TEST: Invalid label definition:", t.LabelDefinition) + } if t.Resource == nil || t.Resource.Identifier == "" { return []byte(`{"error":"internal: invalid resource"}`), log.Error("AWS: Invalid resource:", t.Resource) } + log.Infof("AWS: TaskSnapshot %s: Creating snapshot for Application %s", t.ApplicationTask.UID, t.ApplicationTask.ApplicationUID) conn := t.driver.newEC2Conn() if t.ApplicationTask.When == types.ApplicationStatusDEALLOCATE { // We need to stop the instance before executing snapshot to ensure it will be consistent - input := &ec2.StopInstancesInput{ + input := ec2.StopInstancesInput{ InstanceIds: []string{t.Resource.Identifier}, } - result, err := conn.StopInstances(context.TODO(), input) + log.Infof("AWS: TaskSnapshot %s: Stopping instance %q...", t.ApplicationTask.UID, t.Resource.Identifier) + result, err := conn.StopInstances(context.TODO(), &input) if err != nil { // Do not fail hard here - it's still possible to take snapshot of the instance log.Error("AWS: Error during stopping the instance:", t.Resource.Identifier, err) @@ -90,23 +97,31 @@ func (t *TaskSnapshot) Execute() (result []byte, err error) { } } - input := &ec2.CreateSnapshotsInput{ - InstanceSpecification: &ec2_types.InstanceSpecification{ - ExcludeBootVolume: aws.Bool(!t.Full), - InstanceId: &t.Resource.Identifier, - }, - Description: aws.String("Created by AquariumFish"), - CopyTagsFromSource: ec2_types.CopyTagsFromSourceVolume, + spec := ec2_types.InstanceSpecification{ + ExcludeBootVolume: aws.Bool(!t.Full), + InstanceId: aws.String(t.Resource.Identifier), + } + input := ec2.CreateSnapshotsInput{ + InstanceSpecification: &spec, + Description: aws.String("Created by AquariumFish"), + CopyTagsFromSource: ec2_types.CopyTagsFromSourceVolume, TagSpecifications: []ec2_types.TagSpecification{{ ResourceType: ec2_types.ResourceTypeSnapshot, - Tags: []ec2_types.Tag{{ - Key: aws.String("InstanceId"), - Value: aws.String(t.Resource.Identifier), - }}, + Tags: []ec2_types.Tag{ + { + Key: aws.String("InstanceId"), + Value: aws.String(t.Resource.Identifier), + }, + { + Key: aws.String("ApplicationTask"), + Value: aws.String(t.ApplicationTask.UID.String()), + }, + }, }}, } - resp, err := conn.CreateSnapshots(context.TODO(), input) + log.Debugf("AWS: TaskSnapshot %s: Creating snapshot %q...", t.ApplicationTask.UID) + resp, err := conn.CreateSnapshots(context.TODO(), &input) if err != nil { return []byte{}, log.Errorf("AWS: Unable to create snapshots for instance %s: %v", t.Resource.Identifier, err) } diff --git a/lib/drivers/aws/util.go b/lib/drivers/aws/util.go index 376b7a8..ab20f7b 100644 --- a/lib/drivers/aws/util.go +++ b/lib/drivers/aws/util.go @@ -229,13 +229,43 @@ func (d *Driver) getImageId(conn *ec2.Client, id_name string) (string, error) { }, Owners: d.cfg.AccountIDs, } + p := ec2.NewDescribeImagesPaginator(conn, &req) resp, err := conn.DescribeImages(context.TODO(), &req) if err != nil || len(resp.Images) == 0 { return "", fmt.Errorf("AWS: Unable to locate image with specified name: %v", err) } id_name = aws.ToString(resp.Images[0].ImageId) - return id_name, nil + // Getting the images and find the latest one + var found_id string + var found_time time.Time + for p.HasMorePages() { + resp, err := p.NextPage(context.TODO()) + if err != nil { + return "", fmt.Errorf("AWS: Error during requesting snapshot: %v", err) + } + if len(resp.Images) > 100 { + log.Warnf("AWS: Over 100 images was found for the name %q, could be slow...", id_name) + } + for _, r := range resp.Images { + // Converting from RFC-3339/ISO-8601 format "2024-03-07T15:53:03.000Z" + t, err := time.Parse("2006-01-02T15:04:05.000Z", aws.ToString(r.CreationDate)) + if err != nil { + log.Warnf("AWS: Error during parsing image create time: %v", err) + continue + } + if found_time.Before(t) { + found_id = aws.ToString(r.ImageId) + found_time = t + } + } + } + + if found_id == "" { + return "", fmt.Errorf("AWS: Unable to locate snapshot with specified tag: %s", id_name) + } + + return found_id, nil } // Types are used to calculate some not that obvious values @@ -382,7 +412,7 @@ func (d *Driver) getSnapshotId(conn *ec2.Client, id_tag string) (string, error) tag_key_val := strings.SplitN(id_tag, ":", 2) // Look for VPC with the defined tag over pages - p := ec2.NewDescribeSnapshotsPaginator(conn, &ec2.DescribeSnapshotsInput{ + req := ec2.DescribeSnapshotsInput{ Filters: []types.Filter{ types.Filter{ Name: aws.String("tag:" + tag_key_val[0]), @@ -394,9 +424,10 @@ func (d *Driver) getSnapshotId(conn *ec2.Client, id_tag string) (string, error) }, }, OwnerIds: d.cfg.AccountIDs, - }) + } + p := ec2.NewDescribeSnapshotsPaginator(conn, &req) - // Getting the images to find the latest one + // Getting the snapshots to find the latest one found_id := "" var found_time time.Time for p.HasMorePages() { @@ -408,7 +439,7 @@ func (d *Driver) getSnapshotId(conn *ec2.Client, id_tag string) (string, error) log.Warn("AWS: Over 900 snapshots was found for tag, could be slow:", id_tag) } for _, r := range resp.Snapshots { - if found_time.Before(*r.StartTime) { + if found_time.Before(aws.ToTime(r.StartTime)) { found_id = aws.ToString(r.SnapshotId) found_time = aws.ToTime(r.StartTime) } @@ -427,7 +458,7 @@ func (d *Driver) getProjectCpuUsage(conn *ec2.Client, inst_types []string) (int6 // Here is no way to use some filter, so we're getting them all and after that // checking if the instance is actually starts with type+number. - p := ec2.NewDescribeInstancesPaginator(conn, &ec2.DescribeInstancesInput{ + req := ec2.DescribeInstancesInput{ Filters: []types.Filter{ types.Filter{ Name: aws.String("instance-state-name"), @@ -435,7 +466,8 @@ func (d *Driver) getProjectCpuUsage(conn *ec2.Client, inst_types []string) (int6 Values: []string{"pending", "running", "shutting-down", "stopping", "stopped"}, }, }, - }) + } + p := ec2.NewDescribeInstancesPaginator(conn, &req) // Processing the received instances for p.HasMorePages() { @@ -487,9 +519,10 @@ func (d *Driver) getKeyId(id_alias string) (string, error) { conn := d.newKMSConn() // Look for VPC with the defined tag over pages - p := kms.NewListAliasesPaginator(conn, &kms.ListAliasesInput{ + req := kms.ListAliasesInput{ Limit: aws.Int32(100), - }) + } + p := kms.NewListAliasesPaginator(conn, &req) // Getting the images to find the latest one for p.HasMorePages() { @@ -524,9 +557,10 @@ func (d *Driver) updateQuotas(force bool) error { conn_sq := d.newServiceQuotasConn() // Get the list of quotas - p := servicequotas.NewListServiceQuotasPaginator(conn_sq, &servicequotas.ListServiceQuotasInput{ + req := servicequotas.ListServiceQuotasInput{ ServiceCode: aws.String("ec2"), - }) + } + p := servicequotas.NewListServiceQuotasPaginator(conn_sq, &req) // Processing the received quotas for p.HasMorePages() { @@ -582,15 +616,16 @@ func (d *Driver) triggerHostScrubbing(host_id, instance_type string) (err error) log.Infof("AWS: scrubbing %s: Selected image: %q", host_id, vm_image) // Prepare Instance request information + placement := types.Placement{ + Tenancy: types.TenancyHost, + HostId: aws.String(host_id), + } input := ec2.RunInstancesInput{ ImageId: aws.String(vm_image), InstanceType: types.InstanceType(instance_type), // Set placement to the target host - Placement: &types.Placement{ - Tenancy: types.TenancyHost, - HostId: aws.String(host_id), - }, + Placement: &placement, MinCount: aws.Int32(1), MaxCount: aws.Int32(1), @@ -634,6 +669,48 @@ func (d *Driver) triggerHostScrubbing(host_id, instance_type string) (err error) return nil } +// Will completely delete the image (with associated snapshots) by AMI id +func (d *Driver) deleteImage(conn *ec2.Client, id string) (err error) { + if !strings.HasPrefix(id, "ami-") { + return fmt.Errorf("AWS: Incorrect AMI id: %s", id) + } + log.Debugf("AWS: Deleting the image %s...", id) + + // Look for the image snapshots + req := ec2.DescribeImagesInput{ + ImageIds: []string{id}, + Owners: d.cfg.AccountIDs, + } + resp_img, err := conn.DescribeImages(context.TODO(), &req) + if err != nil || len(resp_img.Images) == 0 { + return fmt.Errorf("AWS: Unable to describe image with specified id %q: %w", id, err) + } + + // Deregister the image + input := ec2.DeregisterImageInput{ImageId: aws.String(id)} + _, err = conn.DeregisterImage(context.TODO(), &input) + if err != nil { + return fmt.Errorf("AWS: Unable to deregister the image %s %q: %w", id, aws.ToString(resp_img.Images[0].Name), err) + } + + // Delete the image snapshots + for _, disk := range resp_img.Images[0].BlockDeviceMappings { + if disk.Ebs == nil || disk.Ebs.SnapshotId == nil { + continue + } + log.Debugf("AWS: Deleting the image %s associated snapshot %s", id, aws.ToString(disk.Ebs.SnapshotId)) + input := ec2.DeleteSnapshotInput{SnapshotId: disk.Ebs.SnapshotId} + _, err_tmp := conn.DeleteSnapshot(context.TODO(), &input) + if err_tmp != nil { + // Do not fail hard to try to delete all the snapshots + log.Errorf("AWS: Unable to delete image %s %q snapshot %s: %v", id, aws.ToString(resp_img.Images[0].Name), aws.ToString(disk.Ebs.SnapshotId), err) + err = err_tmp + } + } + + return err +} + // Returns values for filter to receive only the last year items // For simplicity it's precision is up to month - iterating over days as well generates quite a bit // of complicated logic which is unnecessary for the current usage diff --git a/lib/drivers/task.go b/lib/drivers/task.go index b37f573..fe1a259 100644 --- a/lib/drivers/task.go +++ b/lib/drivers/task.go @@ -25,7 +25,7 @@ type ResourceDriverTask interface { Clone() ResourceDriverTask // Fish provides the task information about the operated items - SetInfo(task *types.ApplicationTask, res *types.Resource) + SetInfo(task *types.ApplicationTask, def *types.LabelDefinition, res *types.Resource) // Run the task operation // <- result - json data with results of operation diff --git a/lib/drivers/test/tasks.go b/lib/drivers/test/tasks.go index d7d3329..0959dbd 100644 --- a/lib/drivers/test/tasks.go +++ b/lib/drivers/test/tasks.go @@ -27,6 +27,7 @@ type TaskSnapshot struct { driver *Driver `json:"-"` *types.ApplicationTask `json:"-"` // Info about the requested task + *types.LabelDefinition `json:"-"` // Info about the used label definition *types.Resource `json:"-"` // Info about the processed resource Full bool `json:"full"` // Make full (all disks including OS image), or just the additional disks snapshot @@ -41,8 +42,9 @@ func (t *TaskSnapshot) Clone() drivers.ResourceDriverTask { return &n } -func (t *TaskSnapshot) SetInfo(task *types.ApplicationTask, res *types.Resource) { +func (t *TaskSnapshot) SetInfo(task *types.ApplicationTask, def *types.LabelDefinition, res *types.Resource) { t.ApplicationTask = task + t.LabelDefinition = def t.Resource = res } @@ -50,6 +52,9 @@ func (t *TaskSnapshot) Execute() (result []byte, err error) { if t.ApplicationTask == nil { return []byte(`{"error":"internal: invalid application task"}`), log.Error("TEST: Invalid application task:", t.ApplicationTask) } + if t.LabelDefinition == nil { + return []byte(`{"error":"internal: invalid label definition"}`), log.Error("TEST: Invalid label definition:", t.LabelDefinition) + } if t.Resource == nil || t.Resource.Identifier == "" { return []byte(`{"error":"internal: invalid resource"}`), log.Error("TEST: Invalid resource:", t.Resource) } diff --git a/lib/fish/fish.go b/lib/fish/fish.go index b3700dd..9269bd8 100644 --- a/lib/fish/fish.go +++ b/lib/fish/fish.go @@ -433,19 +433,21 @@ func (f *Fish) isNodeAvailableForDefinition(def types.LabelDefinition) bool { // Verify node filters because some workload can't be running on all the physical nodes // The node becomes fitting only when all the needed node filter patterns are matched - needed_idents := def.Resources.NodeFilter - current_idents := f.cfg.NodeIdentifiers - for _, needed := range needed_idents { - found := false - for _, value := range current_idents { - // We're validating the pattern on error during label creation, so they should be ok - if found, _ = path.Match(needed, value); found { - break + if def.Resources.NodeFilter != nil && len(def.Resources.NodeFilter) > 0 { + needed_idents := def.Resources.NodeFilter + current_idents := f.cfg.NodeIdentifiers + for _, needed := range needed_idents { + found := false + for _, value := range current_idents { + // We're validating the pattern on error during label creation, so they should be ok + if found, _ = path.Match(needed, value); found { + break + } + } + if !found { + // One of the required node identifiers did not matched the node ones + return false } - } - if !found { - // One of the required node identifiers did not matched the node ones - return false } } // Here all the node filters matched the node identifiers @@ -646,7 +648,7 @@ func (f *Fish) executeApplication(vote types.Vote) error { resource_timeout := res.CreatedAt.Add(resource_lifetime) if app_state.Status == types.ApplicationStatusALLOCATED { if resource_lifetime > 0 { - log.Infof("Fish: Resource %s will be deallocated by timeout in %s (%s)", app.UID, resource_lifetime, resource_timeout) + log.Infof("Fish: Resource of Application %s will be deallocated by timeout in %s (%s)", app.UID, resource_lifetime, resource_timeout) } else { log.Warn("Fish: Resource have no lifetime set and will live until deallocated by user:", app.UID) } @@ -678,8 +680,8 @@ func (f *Fish) executeApplication(vote types.Vote) error { // Execute the existing ApplicationTasks. It will be executed during ALLOCATED or prior // to executing deallocation by DEALLOCATE & RECALLED which right now is useful for - // `snapshot` tasks. - f.executeApplicationTasks(driver, res, app_state.Status) + // `snapshot` and `image` tasks. + f.executeApplicationTasks(driver, &label_def, res, app_state.Status) if app_state.Status == types.ApplicationStatusDEALLOCATE || app_state.Status == types.ApplicationStatusRECALLED { log.Info("Fish: Running Deallocate of the Application:", app.UID) @@ -732,7 +734,7 @@ func (f *Fish) executeApplication(vote types.Vote) error { return nil } -func (f *Fish) executeApplicationTasks(drv drivers.ResourceDriver, res *types.Resource, app_status types.ApplicationStatus) error { +func (f *Fish) executeApplicationTasks(drv drivers.ResourceDriver, def *types.LabelDefinition, res *types.Resource, app_status types.ApplicationStatus) error { // Execute the associated ApplicationTasks if there is some tasks, err := f.ApplicationTaskListByApplicationAndWhen(res.ApplicationUID, app_status) if err != nil { @@ -749,7 +751,7 @@ func (f *Fish) executeApplicationTasks(drv drivers.ResourceDriver, res *types.Re task.Result = util.UnparsedJson(`{"error":"task not availble in driver"}`) } else { // Executing the task - t.SetInfo(&task, res) + t.SetInfo(&task, def, res) result, err := t.Execute() if err != nil { // We're not crashing here because even with error task could have a result diff --git a/lib/log/log.go b/lib/log/log.go index 6aef49a..2e51071 100644 --- a/lib/log/log.go +++ b/lib/log/log.go @@ -53,11 +53,16 @@ func SetVerbosity(level string) error { func InitLoggers() error { flags := log.Lmsgprefix - // Showing short file for debug verbosity - if Verbosity < 2 { - flags |= log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile - } else if UseTimestamp { + // Skip timestamp if not needed + if UseTimestamp { flags |= log.Ldate | log.Ltime + if Verbosity < 2 { + flags |= log.Lmicroseconds + } + } + // Show short file for debug verbosity + if Verbosity < 2 { + flags |= log.Lshortfile } DebugLogger = log.New(os.Stdout, "DEBUG:\t", flags) diff --git a/lib/openapi/api/api_v1.go b/lib/openapi/api/api_v1.go index a753e3e..a4846ff 100644 --- a/lib/openapi/api/api_v1.go +++ b/lib/openapi/api/api_v1.go @@ -356,6 +356,29 @@ func (e *Processor) ApplicationTaskCreatePost(c echo.Context, app_uid types.Appl return c.JSON(http.StatusOK, data) } +func (e *Processor) ApplicationTaskGet(c echo.Context, task_uid types.ApplicationTaskUID) error { + task, err := e.fish.ApplicationTaskGet(task_uid) + if err != nil { + c.JSON(http.StatusBadRequest, H{"message": fmt.Sprintf("Unable to find the Application: %s", task_uid)}) + return fmt.Errorf("Unable to find the ApplicationTask: %s, %w", task_uid, err) + } + + app, err := e.fish.ApplicationGet(task.ApplicationUID) + if err != nil { + c.JSON(http.StatusBadRequest, H{"message": fmt.Sprintf("Unable to find the Application: %s", task.ApplicationUID)}) + return fmt.Errorf("Unable to find the Application: %s, %w", task.ApplicationUID, err) + } + + // Only the owner of the application (or admin) could get the attached task + user := c.Get("user") + if app.OwnerName != user.(*types.User).Name && user.(*types.User).Name != "admin" { + c.JSON(http.StatusBadRequest, H{"message": fmt.Sprintf("Only the owner of Application & admin can get the ApplicationTask")}) + return fmt.Errorf("Only the owner of Application & admin can get the ApplicationTask") + } + + return c.JSON(http.StatusOK, task) +} + func (e *Processor) ApplicationDeallocateGet(c echo.Context, uid types.ApplicationUID) error { app, err := e.fish.ApplicationGet(uid) if err != nil { diff --git a/lib/openapi/openapi.go b/lib/openapi/openapi.go index 14fb279..aeda097 100644 --- a/lib/openapi/openapi.go +++ b/lib/openapi/openapi.go @@ -28,9 +28,9 @@ import ( "strings" "time" - //oapimw "github.com/deepmap/oapi-codegen/pkg/middleware" "github.com/labstack/echo/v4" echomw "github.com/labstack/echo/v4/middleware" + _ "github.com/oapi-codegen/oapi-codegen/v2/pkg/util" "gopkg.in/yaml.v3" "github.com/adobe/aquarium-fish/lib/cluster" diff --git a/lib/openapi/types/label_definitions.go b/lib/openapi/types/label_definitions.go index 63e03a1..2d11fe0 100644 --- a/lib/openapi/types/label_definitions.go +++ b/lib/openapi/types/label_definitions.go @@ -29,9 +29,21 @@ func (ld *LabelDefinitions) Scan(value any) error { } err := json.Unmarshal(bytes, ld) + // Need to make sure the array node filter will not be nil + for i, r := range *ld { + if r.Resources.NodeFilter == nil { + (*ld)[i].Resources.NodeFilter = []string{} + } + } return err } func (ld LabelDefinitions) Value() (driver.Value, error) { + // Need to make sure the array node filter will not be nil + for i, r := range ld { + if r.Resources.NodeFilter == nil { + ld[i].Resources.NodeFilter = []string{} + } + } return json.Marshal(ld) } diff --git a/lib/openapi/types/resources.go b/lib/openapi/types/resources.go index a8f8b21..b735b60 100644 --- a/lib/openapi/types/resources.go +++ b/lib/openapi/types/resources.go @@ -32,10 +32,20 @@ func (r *Resources) Scan(value any) error { } err := json.Unmarshal(bytes, r) + + // Init the value, otherwise will return undesired nil + if err == nil && r.NodeFilter == nil { + r.NodeFilter = []string{} + } + return err } func (r Resources) Value() (driver.Value, error) { + // Init the value, otherwise will return undesired nil + if r.NodeFilter == nil { + r.NodeFilter = []string{} + } return json.Marshal(r) } diff --git a/lib/util/unparsed_json.go b/lib/util/unparsed_json.go index 51bfe19..14e9944 100644 --- a/lib/util/unparsed_json.go +++ b/lib/util/unparsed_json.go @@ -20,8 +20,8 @@ import ( type UnparsedJson string -func (r *UnparsedJson) MarshalJSON() ([]byte, error) { - return []byte(*r), nil +func (r UnparsedJson) MarshalJSON() ([]byte, error) { + return []byte(r), nil } func (r *UnparsedJson) UnmarshalJSON(b []byte) error {