Skip to content

Commit

Permalink
fix: thumbnail file name conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
@guqing
guqing committed Aug 13, 2024
1 parent aff1562 commit 8ddd92f
Show file tree
Hide file tree
Showing 12 changed files with 238 additions and 220 deletions.
8 changes: 8 additions & 0 deletions application/src/main/java/run/halo/app/core/attachment/LocalThumbnailService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.net.URI;
import java.net.URL;
import org.springframework.core.io.Resource;
import org.springframework.lang.NonNull;
import reactor.core.publisher.Mono;
import run.halo.app.core.extension.attachment.LocalThumbnail;
import run.halo.app.infra.ExternalLinkProcessor;
Expand Down Expand Up @@ -47,4 +48,11 @@ public interface LocalThumbnailService {
* @return A {@link Mono} indicates the completion of the deletion.
*/
Mono<Void> delete(URI imageUri);

/**
* Ensures the image URI is an url path if it's an in-site image.
* If it's not an in-site image, it will return directly.
*/
@NonNull
URI ensureInSiteUriIsRelative(URI imageUri);
}
2 changes: 2 additions & 0 deletions application/src/main/java/run/halo/app/core/attachment/ThumbnailService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,6 @@ public interface ThumbnailService {
* @return generated thumbnail uri if success, otherwise empty.
*/
Mono<URI> generate(URI imageUri, ThumbnailSize size);

Mono<Void> delete(URI imageUri);
}
124 changes: 75 additions & 49 deletions application/src/main/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,28 @@
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.isNull;

import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.data.domain.Sort;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import reactor.core.Exceptions;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import run.halo.app.core.attachment.AttachmentRootGetter;
import run.halo.app.core.attachment.AttachmentUtils;
import run.halo.app.core.attachment.LocalThumbnailService;
import run.halo.app.core.attachment.ThumbnailGenerator;
import run.halo.app.core.attachment.ThumbnailSigner;
Expand All @@ -38,6 +40,7 @@
import run.halo.app.extension.Metadata;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.infra.ExternalLinkProcessor;
import run.halo.app.infra.ExternalUrlSupplier;
import run.halo.app.infra.exception.NotFoundException;

Expand All @@ -48,6 +51,7 @@ public class LocalThumbnailServiceImpl implements LocalThumbnailService {
private final AttachmentRootGetter attachmentDirGetter;
private final ReactiveExtensionClient client;
private final ExternalUrlSupplier externalUrlSupplier;
private final ExternalLinkProcessor externalLinkProcessor;

private static Path buildThumbnailStorePath(Path rootPath, String fileName, String year,
ThumbnailSize size) {
Expand All @@ -62,7 +66,7 @@ static String endpointFor(String fileName, String year, ThumbnailSize size) {
return "/upload/thumbnails/%s/w%s/%s".formatted(year, size.getWidth(), fileName);
}

static String getThumbnailFileName(URL imageUrl) {
static String geImageFileName(URL imageUrl) {
var fileName = substringAfterLast(imageUrl.getPath(), "/");
fileName = defaultIfBlank(fileName, randomAlphanumeric(10));
return ThumbnailGenerator.sanitizeFileName(fileName);
Expand Down Expand Up @@ -90,10 +94,7 @@ public Mono<Resource> getThumbnail(String year, ThumbnailSize size, String filen

private Mono<LocalThumbnail> fetchThumbnail(String thumbSignature) {
return client.listBy(LocalThumbnail.class, ListOptions.builder()
.fieldQuery(and(
equal("spec.thumbSignature", thumbSignature),
isNull("metadata.deletionTimestamp"))
)
.fieldQuery(equal("spec.thumbSignature", thumbSignature))
.build(), PageRequestImpl.ofSize(1))
.flatMap(result -> Mono.justOrEmpty(ListResult.first(result)));
}
Expand All @@ -105,7 +106,12 @@ public Mono<Resource> generate(LocalThumbnail thumbnail) {
if (Files.exists(filePath)) {
return Mono.just(new FileSystemResource(filePath));
}
var imageUrl = AttachmentUtils.toUrl(thumbnail.getSpec().getImageUrl());
var imageUrlOpt = toImageUrl(thumbnail.getSpec().getImageUri());
if (imageUrlOpt.isEmpty()) {
return Mono.error(new IllegalStateException(
"Failed to parse image URL，please check external-url configuration."));
}
var imageUrl = imageUrlOpt.get();
var size = thumbnail.getSpec().getSize();
return new ThumbnailGenerator(imageUrl, size, filePath).generate()
.thenReturn((Resource) new FileSystemResource(filePath))
Expand All @@ -120,17 +126,21 @@ public Mono<LocalThumbnail> create(URL imageUrl, ThumbnailSize size) {
Assert.notNull(imageUrl, "Image URL must not be null.");
Assert.notNull(size, "Thumbnail size must not be null.");
var year = getYear();
return ThumbnailInfo.from(imageUrl, size, attachmentDirGetter.get(), year)
.flatMap(thumbInfo -> {
var originalFileName = geImageFileName(imageUrl);
var imageUri = URI.create(imageUrl.toString());
return generateUniqueThumbFileName(originalFileName, year, size)
.flatMap(thumbFileName -> {
var filePath =
buildThumbnailStorePath(attachmentDirGetter.get(), thumbFileName, year, size);
var thumbnail = new LocalThumbnail();
thumbnail.setMetadata(new Metadata());
thumbnail.getMetadata().setGenerateName("thumbnail-");
var thumbnailUri = endpointFor(thumbInfo.fileName(), year, size);
var thumbnailUri = endpointFor(thumbFileName, year, size);
var thumbSignature = ThumbnailSigner.generateSignature(thumbnailUri);
thumbnail.setSpec(new LocalThumbnail.Spec()
.setImageSignature(signatureForImageUri(imageUrl.toString()))
.setFilePath(toRelativeUnixPath(thumbInfo.filePath()))
.setImageUrl(imageUrl.toString())
.setImageSignature(signatureForImageUri(imageUri))
.setFilePath(toRelativeUnixPath(filePath))
.setImageUri(ensureInSiteUriIsRelative(imageUri).toString())
.setSize(size)
.setThumbSignature(thumbSignature)
.setThumbnailUri(thumbnailUri));
Expand All @@ -140,7 +150,7 @@ public Mono<LocalThumbnail> create(URL imageUrl, ThumbnailSize size) {

@Override
public Mono<Void> delete(URI imageUri) {
var signature = signatureForImageUri(imageUri.toString());
var signature = signatureForImageUri(imageUri);
return client.listAll(LocalThumbnail.class, ListOptions.builder()
.fieldQuery(and(
equal("spec.imageSignature", signature),
Expand All @@ -155,6 +165,47 @@ public Mono<Void> delete(URI imageUri) {
.then();
}

@Override
@NonNull
public URI ensureInSiteUriIsRelative(URI imageUri) {
Assert.notNull(imageUri, "Image URI must not be null.");
var externalUrl = externalUrlSupplier.getRaw();
if (externalUrl == null || !isSameOrigin(imageUri, externalUrl)) {
return imageUri;
}
var uriStr = imageUri.toString().replaceFirst("^\\w+://", "");
uriStr = StringUtils.removeStart(uriStr, imageUri.getAuthority());
return URI.create(uriStr);
}

Optional<URL> toImageUrl(String imageUriStr) {
var imageUri = URI.create(imageUriStr);
try {
var url = new URL(externalLinkProcessor.processLink(imageUri.toString()));
return Optional.of(url);
} catch (MalformedURLException e) {
// Ignore
}
return Optional.empty();
}

Mono<String> generateUniqueThumbFileName(String originalFileName, String year,
ThumbnailSize size) {
Assert.notNull(originalFileName, "Original file name must not be null.");
return generateUniqueThumbFileName(originalFileName, originalFileName, year, size);
}

private Mono<String> generateUniqueThumbFileName(String originalFileName, String tryFileName,
String year, ThumbnailSize size) {
var hash = ThumbnailSigner.generateSignature(endpointFor(tryFileName, year, size));
return fetchThumbnail(hash)
.flatMap(thumbnail -> {
// use the original file name to generate a new file name
var newTryFileName = appendRandomSuffix(originalFileName);
return generateUniqueThumbFileName(originalFileName, newTryFileName, year, size);
})
.switchIfEmpty(Mono.just(tryFileName));
}

Path toFilePath(String relativeUnixPath) {
Assert.notNull(relativeUnixPath, "Relative path must not be null.");
Expand Down Expand Up @@ -183,50 +234,25 @@ private Mono<Void> deleteFile(Path path) {
}

/**
* Generate signature for the given image URL.
* <p>if externalUrl is not configured, it will return the signature generated by the image URL
* Generate signature for the given image URI.
* <p>if externalUrl is not configured, it will return the signature generated by the image URI
* directly, otherwise, it will return the signature generated by the relative path of the
* image URL to the external URL.</p>
*/
String signatureForImageUri(String imageUriStr) {
var externalUrl = externalUrlSupplier.getRaw();
var imageUri = URI.create(imageUriStr);
if (externalUrl == null || !isSameOrigin(imageUri, externalUrl)) {
return ThumbnailSigner.generateSignature(imageUriStr);
}
return ThumbnailSigner.generateSignature(imageUri.getPath());
String signatureForImageUri(URI imageUri) {
var uriToSign = ensureInSiteUriIsRelative(imageUri).toString();
return ThumbnailSigner.generateSignature(uriToSign);
}

private boolean isSameOrigin(URI imageUri, URL externalUrl) {
return StringUtils.equals(imageUri.getHost(), externalUrl.getHost())
&& imageUri.getPort() == externalUrl.getPort();
}

record ThumbnailInfo(String fileName, Path filePath) {
public static Mono<ThumbnailInfo> from(URL image, ThumbnailSize size, Path rootPath,
String year) {
return Mono.defer(() -> {
var fileName = getThumbnailFileName(image);
var filePath = generateUniqueFilePath(rootPath, fileName, year, size);
return Mono.just(new ThumbnailInfo(fileName, filePath));
});
}

private static Path generateUniqueFilePath(Path rootPath, String fileName, String year,
ThumbnailSize size) {
var filePath = buildThumbnailStorePath(rootPath, fileName, year, size);
if (Files.exists(filePath)) {
fileName = appendRandomSuffix(fileName);
return generateUniqueFilePath(rootPath, fileName, year, size);
}
return filePath;
}

private static String appendRandomSuffix(String fileName) {
var baseName = StringUtils.substringBeforeLast(fileName, ".");
var extension = substringAfterLast(fileName, ".");
var randomSuffix = randomAlphanumeric(5);
return String.format("%s_%s.%s", baseName, randomSuffix, extension);
}
static String appendRandomSuffix(String fileName) {
var baseName = StringUtils.substringBeforeLast(fileName, ".");
var extension = substringAfterLast(fileName, ".");
var randomSuffix = randomAlphanumeric(6);
return String.format("%s_%s.%s", baseName, randomSuffix, extension);
}
}
48 changes: 43 additions & 5 deletions application/src/main/java/run/halo/app/core/attachment/impl/ThumbnailServiceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,24 @@
package run.halo.app.core.attachment.impl;

import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.startsWith;

import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import reactor.core.publisher.Mono;
import run.halo.app.core.attachment.AttachmentUtils;
import run.halo.app.core.attachment.LocalThumbnailService;
import run.halo.app.core.attachment.ThumbnailProvider;
import run.halo.app.core.attachment.ThumbnailProvider.ThumbnailContext;
import run.halo.app.core.attachment.ThumbnailService;
import run.halo.app.core.attachment.ThumbnailSigner;
import run.halo.app.core.attachment.ThumbnailSize;
import run.halo.app.core.extension.attachment.Thumbnail;
import run.halo.app.extension.ListOptions;
Expand All @@ -30,6 +36,8 @@ public class ThumbnailServiceImpl implements ThumbnailService {
private final ExtensionGetter extensionGetter;
private final ReactiveExtensionClient client;
private final ExternalLinkProcessor externalLinkProcessor;
private final ThumbnailProvider thumbnailProvider;
private final LocalThumbnailService localThumbnailService;

@Override
public Mono<URI> generate(URI imageUri, ThumbnailSize size) {
Expand All @@ -38,7 +46,7 @@ public Mono<URI> generate(URI imageUri, ThumbnailSize size) {
return Mono.empty();
}
var imageUrl = imageUrlOpt.get();
return fetchThumbnail(imageUrl, size)
return fetchThumbnail(imageUri, size)
.map(thumbnail -> URI.create(thumbnail.getSpec().getThumbnailUri()))
.switchIfEmpty(create(imageUrl, size))
.onErrorResume(Throwable.class, e -> {
Expand All @@ -47,6 +55,29 @@ public Mono<URI> generate(URI imageUri, ThumbnailSize size) {
});
}

@Override
public Mono<Void> delete(URI imageUri) {
Assert.notNull(imageUri, "Image uri must not be null");
Mono<Void> deleteMono;
if (imageUri.isAbsolute()) {
deleteMono = thumbnailProvider.delete(AttachmentUtils.toUrl(imageUri));
} else {
// Local thumbnails maybe a relative path, so we need to process it.
deleteMono = localThumbnailService.delete(imageUri);
}
return deleteMono.then(deleteThumbnailRecord(imageUri));
}

private Mono<Void> deleteThumbnailRecord(URI imageUri) {
var imageHash = signatureFor(imageUri);
var listOptions = ListOptions.builder()
.fieldQuery(startsWith(Thumbnail.ID_INDEX, Thumbnail.idIndexFunc(imageHash, "")))
.build();
return client.listAll(Thumbnail.class, listOptions, Sort.unsorted())
.flatMap(client::delete)
.then();
}

Optional<URL> toImageUrl(URI imageUri) {
try {
if (imageUri.isAbsolute()) {
Expand All @@ -65,6 +96,8 @@ Mono<URI> create(URL imageUrl, ThumbnailSize size) {
.imageUrl(imageUrl)
.size(size)
.build();
var imageUri =
localThumbnailService.ensureInSiteUriIsRelative(URI.create(imageUrl.toString()));
return extensionGetter.getEnabledExtensions(ThumbnailProvider.class)
.filterWhen(provider -> provider.supports(context))
.next()
Expand All @@ -76,16 +109,21 @@ Mono<URI> create(URL imageUrl, ThumbnailSize size) {
thumb.setSpec(new Thumbnail.Spec()
.setSize(size)
.setThumbnailUri(uri.toString())
.setImageUrl(imageUrl)
.setImageSignature(Thumbnail.signatureFor(imageUrl.toString()))
.setImageUri(imageUri.toString())
.setImageSignature(signatureFor(imageUri))
);
return client.create(thumb)
.thenReturn(uri);
});
}

private Mono<Thumbnail> fetchThumbnail(URL imageUrl, ThumbnailSize size) {
var imageHash = Thumbnail.signatureFor(imageUrl.toString());
private String signatureFor(URI imageUri) {
var uri = localThumbnailService.ensureInSiteUriIsRelative(imageUri);
return ThumbnailSigner.generateSignature(uri.toString());
}

Mono<Thumbnail> fetchThumbnail(URI imageUri, ThumbnailSize size) {
var imageHash = signatureFor(imageUri);
var id = Thumbnail.idIndexFunc(imageHash, size.name());
return client.listBy(Thumbnail.class, ListOptions.builder()
.fieldQuery(equal(Thumbnail.ID_INDEX, id))
Expand Down
10 changes: 9 additions & 1 deletion application/src/main/java/run/halo/app/core/attachment/reconciler/AttachmentReconciler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.net.URI;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -48,7 +49,9 @@ public Result reconcile(Request request) {
return;
}
// add finalizer
addFinalizers(attachment.getMetadata(), Set.of(Constant.FINALIZER_NAME));
if (addFinalizers(attachment.getMetadata(), Set.of(Constant.FINALIZER_NAME))) {
client.update(attachment);
}

var annotations = attachment.getMetadata().getAnnotations();
if (annotations != null) {
Expand Down Expand Up @@ -114,6 +117,11 @@ void updateStatus(String attachmentName, AttachmentStatus status) {
}

void cleanUpResources(Attachment attachment) {
Optional.ofNullable(attachment.getStatus())
.map(AttachmentStatus::getPermalink)
.map(URI::create)
.ifPresent(uri -> thumbnailService.delete(uri).block());

attachmentService.delete(attachment).block();
}
}
Loading

0 comments on commit 8ddd92f

Please sign in to comment.