diff --git a/.gitignore b/.gitignore
index f6c8fac..c9b19e5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,5 +31,6 @@ go.work
 
 # End of https://www.toptal.com/developers/gitignore/api/go
 
+.vscode
 .env
 terraform-backend
diff --git a/docker-compose.yml b/docker-compose.yml
index a73e393..7309a5a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,6 +26,14 @@ services:
       POSTGRES_PASSWORD: postgres
     ports:
       - "5432:5432"
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ROOT_USER: root
+      MINIO_ROOT_PASSWORD: password
+    command: server /storage
+    ports:
+      - "9000:9000"
 
 volumes:
   states:
diff --git a/pkg/server/handler.go b/pkg/server/handler.go
index 33709d7..f90a119 100644
--- a/pkg/server/handler.go
+++ b/pkg/server/handler.go
@@ -176,5 +176,13 @@ func Post(r *http.Request, w http.ResponseWriter, state *terraform.State, body [
 
 func Delete(w http.ResponseWriter, state *terraform.State, store storage.Storage) {
 	log.Debugf("delete state with id %s", state.ID)
-	HTTPResponse(w, http.StatusNotImplemented, "Delete state is not implemented")
+
+	err := store.DeleteState(state.ID)
+	if err != nil {
+		log.Warnf("failed to delete state with id %s: %v", state.ID, err)
+		HTTPResponse(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	HTTPResponse(w, http.StatusOK, "")
 }
diff --git a/pkg/storage/filesystem/filesystem.go b/pkg/storage/filesystem/filesystem.go
index c0776f0..365b96c 100644
--- a/pkg/storage/filesystem/filesystem.go
+++ b/pkg/storage/filesystem/filesystem.go
@@ -30,7 +30,7 @@ func (f *FileSystemStorage) GetName() string {
 }
 
 func (f *FileSystemStorage) SaveState(s *terraform.State) error {
-	return os.WriteFile(fmt.Sprintf("%s/%s.tfstate", f.directory, s.ID), s.Data, 0600)
+	return os.WriteFile(f.getFileName(s.ID), s.Data, 0600)
 }
 
 func (f *FileSystemStorage) GetState(id string) (*terraform.State, error) {
@@ -54,6 +54,10 @@ func (f *FileSystemStorage) GetState(id string) (*terraform.State, error) {
 	}, nil
 }
 
+func (f *FileSystemStorage) DeleteState(id string) error {
+	return os.Remove(f.getFileName(id))
+}
+
 func (f *FileSystemStorage) getFileName(id string) string {
 	return fmt.Sprintf("%s/%s.tfstate", f.directory, id)
 }
diff --git a/pkg/storage/filesystem/filesystem_test.go b/pkg/storage/filesystem/filesystem_test.go
new file mode 100644
index 0000000..90a5c5c
--- /dev/null
+++ b/pkg/storage/filesystem/filesystem_test.go
@@ -0,0 +1,16 @@
+package filesystem
+
+import (
+	"testing"
+
+	"github.com/nimbolus/terraform-backend/pkg/storage/util"
+)
+
+func TestStorage(t *testing.T) {
+	s, err := NewFileSystemStorage("./storage")
+	if err != nil {
+		t.Error(err)
+	}
+
+	util.StorageTest(t, s)
+}
diff --git a/pkg/storage/postgres/postgres.go b/pkg/storage/postgres/postgres.go
index 719d394..d2d4bdf 100644
--- a/pkg/storage/postgres/postgres.go
+++ b/pkg/storage/postgres/postgres.go
@@ -76,3 +76,7 @@ func (p *PostgresStorage) GetState(id string) (*terraform.State, error) {
 
 	return s, nil
 }
+
+func (p *PostgresStorage) DeleteState(id string) error {
+	return p.db.QueryRow(`DELETE FROM `+p.table+` WHERE state_id = $1`, id).Err()
+}
diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go
index 5b2535c..38766e6 100644
--- a/pkg/storage/s3/s3.go
+++ b/pkg/storage/s3/s3.go
@@ -24,13 +24,15 @@ func NewS3Storage(endpoint, bucket, accessKey, secretKey string, useSSL bool) (*
 		Secure: useSSL,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize minio client: %v", err)
+		return nil, fmt.Errorf("failed to initialize minio client: %w", err)
 	}
 
 	if exists, err := client.BucketExists(context.Background(), bucket); err != nil {
-		return nil, fmt.Errorf("failed to check for bucket: %v", err)
+		return nil, fmt.Errorf("failed to check for bucket: %w", err)
 	} else if !exists {
-		return nil, fmt.Errorf("bucket does not exist")
+		if err = client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{}); err != nil {
+			return nil, fmt.Errorf("bucket does not exist and creation failed: %w", err)
+		}
 	}
 
 	return &S3Storage{
@@ -74,6 +76,10 @@ func (s *S3Storage) GetState(id string) (*terraform.State, error) {
 	return state, nil
 }
 
+func (s *S3Storage) DeleteState(id string) error {
+	return s.client.RemoveObject(context.Background(), s.bucket, getObjectName(id), minio.RemoveObjectOptions{})
+}
+
 func getObjectName(id string) string {
 	return fmt.Sprintf("%s.tfstate", id)
 }
diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go
new file mode 100644
index 0000000..75240dc
--- /dev/null
+++ b/pkg/storage/s3/s3_test.go
@@ -0,0 +1,19 @@
+//go:build integration || s3
+// +build integration s3
+
+package s3
+
+import (
+	"testing"
+
+	"github.com/nimbolus/terraform-backend/pkg/storage/util"
+)
+
+func TestStorage(t *testing.T) {
+	s, err := NewS3Storage("localhost:9000", "tf-backend-integration-test", "root", "password", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	util.StorageTest(t, s)
+}
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 1afabc4..e055173 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -8,4 +8,5 @@ type Storage interface {
 	GetName() string
 	SaveState(s *terraform.State) error
 	GetState(id string) (*terraform.State, error)
+	DeleteState(id string) error
 }
diff --git a/pkg/storage/util/storagetest.go b/pkg/storage/util/storagetest.go
index af5e043..f5f11e4 100644
--- a/pkg/storage/util/storagetest.go
+++ b/pkg/storage/util/storagetest.go
@@ -48,4 +48,9 @@ func StorageTest(t *testing.T, s storage.Storage) {
 	if string(state.Data) != string(savedState.Data) {
 		t.Errorf("state data does not match")
 	}
+
+	err = s.DeleteState(state.ID)
+	if err != nil {
+		t.Error(err)
+	}
 }