Skip to content

Feat/cache imp #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 13, 2025
Merged

Feat/cache imp #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ee57815
feat: per node caching
VineeTagarwaL-code Jun 13, 2025
a1286f6
feat: per node caching
VineeTagarwaL-code Jun 13, 2025
bfd4995
feat: save pre-translate on change added
VineeTagarwaL-code Jun 13, 2025
3019623
Delete example/translation-demo-nextjs/package-lock.json
VineeTagarwaL-code Jun 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion example/translation-demo-nextjs/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"react": "^19.0.0",
"react-dom": "^19.0.0",
"tailwind-merge": "^3.3.0",
"translation-widget": "^1.0.1"
"translation-widget": "^1.0.2"
},
"devDependencies": {
"@eslint/eslintrc": "^3",
Expand Down
8 changes: 4 additions & 4 deletions example/translation-demo-nextjs/yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2951,10 +2951,10 @@ to-regex-range@^5.0.1:
dependencies:
is-number "^7.0.0"

translation-widget@^1.0.1:
version "1.0.1"
resolved "https://registry.npmjs.org/translation-widget/-/translation-widget-1.0.1.tgz"
integrity sha512-ADIA0vZG0srHAlgv1fxe4HhnSL1S5b/o2gQT6Z7t+l3AoZ1WqazXjtOJ8BjIhloJEJlOdEVNToMydbAk/mPTNA==
translation-widget@^1.0.2:
version "1.0.2"
resolved "https://registry.npmjs.org/translation-widget/-/translation-widget-1.0.2.tgz"
integrity sha512-g9Wc654qm7evRAHpjLQzylkdjY1ZSLYcQC9fWbGr6fBzo6Jpp6ZJTsRyHIQXYQRd11kEcq0GI47HdRkcBdFYqw==
dependencies:
lz-string "^1.5.0"

Expand Down
7 changes: 3 additions & 4 deletions index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,10 @@

<body>
<header class="tw-header">
<h1>Run</h1>
<nav class="tw-nav">
<p> 😀</p>
<a href="#home">Home</a>
<a href="#products">Products</a>
<a href="#products">Product</a>
<a href="#about">About</a>
<a href="#contact">Contact</a>
</nav>
Expand Down Expand Up @@ -113,11 +112,11 @@ <h2>Customer Stories</h2>
console.log(res);
}, (err)=>{
console.log(err);
})" class="tw-button notranslate">Translate to hindi</button>
})" class="tw-button notranslate">Translat to hindi</button>

<button onclick="window.resetTranslation('en', (res)=>{
console.log(res);
})">Reset Translation</button>
})">Reset Translio</button>

</main>
<script defer src="http://localhost:5173/dist/index.min.js"></script>
Expand Down
66 changes: 50 additions & 16 deletions src/lib/storage/localstorage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,9 @@ export class LocalStorageWrapper {
this.prefix = prefix;
}

getKey(hash: string, url: string, targetLang: string): string {
// get rid of query params
getPageKey(url: string, targetLang: string): string {
const urlWithoutQuery = url.split("?")[0];
// Only encode the URL, not the whole key
return `${hash}-${encodeURIComponent(urlWithoutQuery)}-${targetLang}`;
return `${this.prefix}page-${encodeURIComponent(urlWithoutQuery)}-${targetLang}`;
}

private shouldCompress(value: string): boolean {
Expand All @@ -39,12 +37,10 @@ export class LocalStorageWrapper {
}

getItem(key: string): any {
const prefixedKey = this.prefix + key;
const item = localStorage.getItem(prefixedKey);
const item = localStorage.getItem(key);
if (!item) return null;

try {
// Check if the item is compressed
const decompressed = item.startsWith(this.COMPRESSION_MARKER) ? this.decompress(item.slice(this.COMPRESSION_MARKER.length)) : item;
return JSON.parse(decompressed);
} catch (e) {
Expand All @@ -54,31 +50,69 @@ export class LocalStorageWrapper {
}

setItem(key: string, value: any): void {
const prefixedKey = this.prefix + key;
const stringified = JSON.stringify(value);

// Use requestIdleCallback to defer compression if available
const storeValue = () => {
try {
const finalValue = this.shouldCompress(stringified) ? `${this.COMPRESSION_MARKER}${this.compress(stringified)}` : stringified;
localStorage.setItem(prefixedKey, finalValue);
localStorage.setItem(key, finalValue);
} catch (error) {
console.error("Error storing item:", error);
// Fallback to storing uncompressed value
localStorage.setItem(prefixedKey, stringified);
localStorage.setItem(key, stringified);
}
};

if (typeof requestIdleCallback !== "undefined") {
requestIdleCallback(() => storeValue());
} else {
setTimeout(storeValue, 0);
}
}

// Get translation for a node from the page cache (object of node hashes)
getNodeTranslation(nodeHash: string, url: string, targetLang: string): { o: string; t: string } | null {
const pageKey = this.getPageKey(url, targetLang);
const translations: { [key: string]: { o: string; t: string } } = this.getItem(pageKey) || {};
const nodeKey = `jss-node-${nodeHash}`;
return translations[nodeKey] || null;
}

// Store translation for a node in the page cache (object of node hashes)
setNodeTranslation(nodeHash: string, url: string, targetLang: string, translation: { o: string; t: string }): void {
const pageKey = this.getPageKey(url, targetLang);
let translations: { [key: string]: { o: string; t: string } }[] = this.getItem(pageKey) || [];
const nodeKey = `${nodeHash}`;
translations.push({ [nodeKey]: translation });
this.setItem(pageKey, translations);
}

setBatchNodeTranslationsArray(
url: string,
targetLang: string,
batch: Array<{ [key: string]: { o: string; t: string } }>
): void {
const pageKey = this.getPageKey(url, targetLang);
const existing: Array<{ [key: string]: { o: string; t: string } }> = this.getItem(pageKey) || [];

// Convert existing to a map for fast lookup
const map: { [key: string]: { o: string; t: string } } = {};
existing.forEach(obj => {
const key = Object.keys(obj)[0];
map[key] = obj[key];
});

// Add/overwrite with new batch
batch.forEach(obj => {
const key = Object.keys(obj)[0];
map[key] = obj[key];
});

// Convert back to array of objects
const merged = Object.keys(map).map(key => ({ [key]: map[key] }));

this.setItem(pageKey, merged);
}

removeItem(key: string): void {
const prefixedKey = this.prefix + key;
localStorage.removeItem(prefixedKey);
localStorage.removeItem(key);
}

clear(): void {
Expand Down
15 changes: 14 additions & 1 deletion src/utils/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,17 @@ const getUserLanguage = () => {
return userLanguage?.code || "en";
};

export { generateHashForContent, getVisibleTextContent, removeEmojis, getUserLanguage };
function generateNodeHash(text: string): string {
const normalizedText = text.replace(/\s+/g, " ").trim().toLocaleLowerCase();
return murmurhash3_32_gc(normalizedText, 42).toString(16);
}

function generateChunkHash(texts: string[]): string {
const content = texts
.map(text => text.replace(/\s+/g, " ").trim().toLocaleLowerCase())
.join(" ")
.trim();
return murmurhash3_32_gc(content.toLowerCase(), 42).toString(16);
}

export { generateHashForContent, generateNodeHash, generateChunkHash, getVisibleTextContent, removeEmojis, getUserLanguage };
128 changes: 55 additions & 73 deletions src/widget/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { languages } from "../constants/languages";
import { BATCH_SIZE, DEFAULT_CONFIG } from "../constants";
import type { Language, TranslationConfig, WidgetElements, TranslationResult } from "../types";
import widgetTemplate from "../templates/html/widget.html?raw";
import { generateHashForContent, getUserLanguage, removeEmojis } from "../utils/utils";
import { generateHashForContent, generateNodeHash, getUserLanguage, removeEmojis } from "../utils/utils";
import { CACHE_PREFIX } from "../constants";
import { LocalStorageWrapper } from "../lib/storage/localstorage";

Expand Down Expand Up @@ -414,16 +414,18 @@ export class TranslationWidget {

// Initialize cache for storing translations
const cache = new LocalStorageWrapper(CACHE_PREFIX);
// Generate a hash for the current content to use as a cache key
let hash = generateHashForContent(nodes);

// Arrays to store nodes and texts for each batch
const allBatchNodes: { element: HTMLElement; text: string }[][] = [];
const allBatchTexts: string[][] = [];
const allBatchNodeHashes: string[][] = [];

// Prepare batches by filtering nodes that need translation
batches.forEach((batch) => {
const textsToTranslate: string[] = [];
const batchNodes: { element: HTMLElement; text: string }[] = [];
const batchNodeHashes: string[] = [];

batch.forEach((node) => {
const parent = node.element;
if (!parent) return;
Expand All @@ -443,103 +445,83 @@ export class TranslationWidget {

// Add text and node to the batch if valid
if (textToTranslate) {
const nodeHash = generateNodeHash(textToTranslate);
// Use the new cache structure: array of objects
const cacheArray = cache.getItem(cache.getPageKey(window.location.href, targetLang)) || [];
const found = cacheArray.find((obj: Record<string, { o: string; t: string }>) => Object.prototype.hasOwnProperty.call(obj, nodeHash));
const cachedTranslation = found ? found[nodeHash] : null;

if (cachedTranslation) {
// Use cached translation
if (this.lastRequestedLanguage === targetLang) {
const originalText = cachedTranslation.o;
const translatedText = cachedTranslation.t;
const originalFontSize = parent.getAttribute("data-original-font-size") || "16px";
const newFontSize = this.calculateFontSize(translatedText, originalFontSize, originalText);
parent.style.fontSize = newFontSize;
parent.textContent = translatedText;
}
return;
}

textsToTranslate.push(textToTranslate.trim());
batchNodes.push(node);
batchNodeHashes.push(nodeHash);
}
});
allBatchNodes.push(batchNodes);
allBatchTexts.push(textsToTranslate);
});

// Filter out empty batches
const nonEmptyBatchNodes: { element: HTMLElement; text: string }[][] = [];
const nonEmptyBatchTexts: string[][] = [];
allBatchTexts.forEach((texts, i) => {
if (texts.length > 0) {
nonEmptyBatchTexts.push(texts);
nonEmptyBatchNodes.push(allBatchNodes[i]);
if (textsToTranslate.length > 0) {
allBatchNodes.push(batchNodes);
allBatchTexts.push(textsToTranslate);
allBatchNodeHashes.push(batchNodeHashes);
}
});

// Check cache for existing translations
const key = cache.getKey(hash, window.location.href, targetLang);
const cachedTranslations = cache.getItem(key);
if (cachedTranslations && cachedTranslations[0]) {
const fullTranslations = cachedTranslations[0];
// Update DOM if this is the most recent request
// If no nodes need translation, we're done
if (allBatchTexts.length === 0) {
if (this.lastRequestedLanguage === targetLang) {
nodes.forEach((node, idx) => {
const parent = node.element;
if (parent) {
const originalText = parent.getAttribute("data-original-text") || "";
const originalFontSize = parent.getAttribute("data-original-font-size") || "16px";
const newFontSize = this.calculateFontSize(fullTranslations[idx], originalFontSize, originalText);
parent.style.fontSize = newFontSize;
parent.textContent = fullTranslations[idx];
}
});
this.isTranslated = true;
this.updateResetButtonVisibility();
}
return;
}

// Translate all batches in parallel
const allTranslatedTexts = await Promise.all(nonEmptyBatchTexts.map((texts) => this.translationService.translateBatchText(texts, targetLang)));
const allTranslatedTexts = await Promise.all(
allBatchTexts.map((texts) => this.translationService.translateBatchText(texts, targetLang))
);

// If no translations were made, update UI state
if (allTranslatedTexts.length === 0) {
if (this.lastRequestedLanguage === targetLang) {
this.isTranslated = true;
this.updateResetButtonVisibility();
}
return;
}
// Process translated batches
const batchArray: Array<{ [key: string]: { o: string; t: string } }> = [];
allTranslatedTexts.forEach((translations, batchIndex) => {
const batchNodes = allBatchNodes[batchIndex];
const batchNodeHashes = allBatchNodeHashes[batchIndex];

// Check if all batches failed
const allBatchesFailed = allTranslatedTexts.every((translations, batchIndex) => {
const originalTexts = nonEmptyBatchTexts[batchIndex];
return translations.every((translation, index) => translation === originalTexts[index]);
});
batchNodes.forEach((node, nodeIndex) => {
const parent = node.element;
if (parent) {
const originalText = node.text;
const translatedText = translations[nodeIndex];
const nodeHash = batchNodeHashes[nodeIndex];

if (allBatchesFailed) {
console.warn("All translations failed, not caching results");
throw new Error("All translation batches failed");
}
// Collect the translation for batch saving
batchArray.push({ [`${nodeHash}`]: { o: originalText, t: translatedText } });

// Build a full translation array for all nodes
const fullTranslations: string[] = [];
nodes.forEach((node, nodeIdx) => {
const parent = node.element;
// Check if this node was included in the API call
const batchIdx = nonEmptyBatchNodes.findIndex((batch) => batch.some(n => n.element === parent));
if (batchIdx !== -1) {
// This node was translated in this batch
const textIdx = nonEmptyBatchNodes[batchIdx].findIndex(n => n.element === parent);
const translatedText = allTranslatedTexts[batchIdx][textIdx];
fullTranslations[nodeIdx] = translatedText;

// Update DOM if this is the most recent request
if (this.lastRequestedLanguage === targetLang) {
// Apply font size adjustment
if (parent) {
const originalText = parent.getAttribute("data-original-text") || "";
// Update DOM if this is the most recent request
if (this.lastRequestedLanguage === targetLang) {
const originalFontSize = parent.getAttribute("data-original-font-size") || "16px";
const newFontSize = this.calculateFontSize(translatedText, originalFontSize, originalText);
parent.style.fontSize = newFontSize;
parent.textContent = translatedText;
}
parent.textContent = translatedText;
}
} else if (parent && parent.getAttribute("data-translated-lang") === targetLang) {
// Already translated, use current text
fullTranslations[nodeIdx] = parent.textContent || "";
} else {
fullTranslations[nodeIdx] = parent.textContent || "";
}
});
});

// Cache the translations
cache.setItem(key, [fullTranslations]);
// Save all translations in one batch
if (batchArray.length > 0) {
cache.setBatchNodeTranslationsArray(window.location.href, targetLang, batchArray);
}

// Update UI state if this is the most recent request
if (this.lastRequestedLanguage === targetLang) {
Expand Down