Skip to content
This repository has been archived by the owner on Nov 14, 2024. It is now read-only.

Complement QoL changes #2663

Merged
merged 8 commits into from
Aug 23, 2022
Merged

Complement QoL changes #2663

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
86a19d0
Add option to specify keySize when generating keys
S7evinK Aug 22, 2022
2968aa6
Update Complement dockerfiles to use buildkit and exec for the process
S7evinK Aug 22, 2022
d1062e5
Use larger keySize when running tests
S7evinK Aug 22, 2022
025f552
Merge branch 'main' into s7evink/complement
S7evinK Aug 23, 2022
dbd402c
Enable BuildKit when building complement images
S7evinK Aug 23, 2022
76748ae
Merge branch 's7evink/complement' of github.com:matrix-org/dendrite i…
S7evinK Aug 23, 2022
35098a5
Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/co…
S7evinK Aug 23, 2022
35fd077
Use larger key size to avoid handshake errors
S7evinK Aug 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions build/scripts/Complement.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2

FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /build
Expand All @@ -8,14 +10,12 @@ RUN mkdir /dendrite

# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
COPY go.mod .
COPY go.sum .
RUN go mod download

COPY . .
RUN go build -o /dendrite ./cmd/dendrite-monolith-server
RUN go build -o /dendrite ./cmd/generate-keys
RUN go build -o /dendrite ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server

WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
Expand All @@ -26,7 +26,7 @@ EXPOSE 8008 8448

# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
CMD ./generate-keys -keysize 64 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What difference does this make in practice?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seeming 5m 49s on this PR, vs 7m 5s on the last main run.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For Postgres it's almost 5m faster (8m 19s vs 12m 49s)

23 changes: 11 additions & 12 deletions build/scripts/ComplementLocal.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2

# A local development Complement dockerfile, to be used with host mounts
# /cache -> Contains the entire dendrite code at Dockerfile build time. Builds binaries but only keeps the generate-* ones. Pre-compilation saves time.
# /dendrite -> Host-mounted sources
Expand All @@ -9,11 +11,10 @@
FROM golang:1.18-stretch
RUN apt-get update && apt-get install -y sqlite3

WORKDIR /runtime

ENV SERVER_NAME=localhost
EXPOSE 8008 8448

WORKDIR /runtime
# This script compiles Dendrite for us.
RUN echo '\
#!/bin/bash -eux \n\
Expand All @@ -29,25 +30,23 @@ RUN echo '\
RUN echo '\
#!/bin/bash -eu \n\
./generate-keys --private-key matrix_key.pem \n\
./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-keys -keysize 64 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
' > run.sh && chmod +x run.sh


WORKDIR /cache
# Pre-download deps; we don't need to do this if the GOPATH is mounted.
COPY go.mod .
COPY go.sum .
RUN go mod download

# Build the monolith in /cache - we won't actually use this but will rely on build artifacts to speed
# up the real compilation. Build the generate-* binaries in the true /runtime locations.
# If the generate-* source is changed, this dockerfile needs re-running.
COPY . .
RUN go build ./cmd/dendrite-monolith-server && go build -o /runtime ./cmd/generate-keys && go build -o /runtime ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /runtime ./cmd/generate-config && \
go build -o /runtime ./cmd/generate-keys


WORKDIR /runtime
CMD /runtime/compile.sh && /runtime/run.sh
CMD /runtime/compile.sh && exec /runtime/run.sh
20 changes: 10 additions & 10 deletions build/scripts/ComplementPostgres.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2

FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
Expand Down Expand Up @@ -26,14 +28,12 @@ RUN mkdir /dendrite

# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
COPY go.mod .
COPY go.sum .
RUN go mod download

COPY . .
RUN go build -o /dendrite ./cmd/dendrite-monolith-server
RUN go build -o /dendrite ./cmd/generate-keys
RUN go build -o /dendrite ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server

WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
Expand All @@ -45,10 +45,10 @@ EXPOSE 8008 8448

# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
CMD /build/run_postgres.sh && ./generate-keys --keysize 64 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
# Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CC: @MadLittleMods I think you were complaining about longer build/execution times, iirc?

Yes! Especially with Synapse, matrix-org/synapse#13204

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like Synapse is already PID 1 inside the container and receives the SIGTERM request, so this is not the issue. :/

5 changes: 3 additions & 2 deletions cmd/generate-keys/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ var (
authorityCertFile = flag.String("tls-authority-cert", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
authorityKeyFile = flag.String("tls-authority-key", "", "Optional: Create TLS certificate/keys based on this CA authority. Useful for integration testing.")
serverName = flag.String("server", "", "Optional: Create TLS certificate/keys with this domain name set. Useful for integration testing.")
keySize = flag.Int("keysize", 4096, "Optional: Create TLS RSA private key with the given key size")
)

func main() {
Expand All @@ -58,12 +59,12 @@ func main() {
log.Fatal("Zero or both of --tls-key and --tls-cert must be supplied")
}
if *authorityCertFile == "" && *authorityKeyFile == "" {
if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile); err != nil {
if err := test.NewTLSKey(*tlsKeyFile, *tlsCertFile, *keySize); err != nil {
panic(err)
}
} else {
// generate the TLS cert/key based on the authority given.
if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile); err != nil {
if err := test.NewTLSKeyWithAuthority(*serverName, *tlsKeyFile, *tlsCertFile, *authorityKeyFile, *authorityCertFile, *keySize); err != nil {
panic(err)
}
}
Expand Down
2 changes: 1 addition & 1 deletion test/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func ListenAndServe(t *testing.T, router http.Handler, withTLS bool) (apiURL str
if withTLS {
certFile := filepath.Join(t.TempDir(), "dendrite.cert")
keyFile := filepath.Join(t.TempDir(), "dendrite.key")
err = NewTLSKey(keyFile, certFile)
err = NewTLSKey(keyFile, certFile, 1024)
if err != nil {
t.Errorf("failed to make TLS key: %s", err)
return
Expand Down
12 changes: 6 additions & 6 deletions test/keys.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ func NewMatrixKey(matrixKeyPath string) (err error) {

const certificateDuration = time.Hour * 24 * 365 * 10

func generateTLSTemplate(dnsNames []string) (*rsa.PrivateKey, *x509.Certificate, error) {
priv, err := rsa.GenerateKey(rand.Reader, 4096)
func generateTLSTemplate(dnsNames []string, bitSize int) (*rsa.PrivateKey, *x509.Certificate, error) {
priv, err := rsa.GenerateKey(rand.Reader, bitSize)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -118,8 +118,8 @@ func writePrivateKey(tlsKeyPath string, priv *rsa.PrivateKey) error {
}

// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
priv, template, err := generateTLSTemplate(nil)
func NewTLSKey(tlsKeyPath, tlsCertPath string, keySize int) error {
priv, template, err := generateTLSTemplate(nil, keySize)
if err != nil {
return err
}
Expand All @@ -136,8 +136,8 @@ func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
return writePrivateKey(tlsKeyPath, priv)
}

func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string) error {
priv, template, err := generateTLSTemplate([]string{serverName})
func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string, keySize int) error {
priv, template, err := generateTLSTemplate([]string{serverName}, keySize)
if err != nil {
return err
}
Expand Down