Skip to content

Commit

Permalink
Merge pull request #1868 from firebase/next
Browse files Browse the repository at this point in the history
  • Loading branch information
pr-Mais authored Nov 30, 2023
2 parents 0d56fe4 + 7da6e53 commit 01d9102
Show file tree
Hide file tree
Showing 22 changed files with 545 additions and 13,305 deletions.
5 changes: 3 additions & 2 deletions _emulator/extensions/storage-resize-images.env.local
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
LOCATION=europe-west2
IMG_BUCKET=${STORAGE_BUCKET}
IMG_SIZES=200x200
IMG_SIZES=300x300
DELETE_ORIGINAL_FILE=true
MAKE_PUBLIC=true
RESIZED_IMAGES_PATH=thumbnails
FAILED_IMAGES_PATH=failed
IMAGE_TYPE=webp
IS_ANIMATED=true
FUNCTION_MEMORY=1024
DO_BACKFILL=false
SHARP_OPTIONS='{"fit":"cover", "position": "top", "animated": false}'
4 changes: 4 additions & 0 deletions firestore-bigquery-export/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## Version 0.1.42

fix - correctly extract timestamps from firestore fields to partition columns

## Version 0.1.41

fix - rollback backfill feature
Expand Down
8 changes: 0 additions & 8 deletions firestore-bigquery-export/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,14 +146,6 @@ To install an extension, your project must be on the [Blaze (pay as you go) plan

* Use new query syntax for snapshots: If enabled, snapshots will be generated with the new query syntax, which should be more performant, and avoid potential resource limitations.

* Import existing Firestore documents into BigQuery?: Do you want to import existing documents from your Firestore collection into BigQuery? These documents will have each have a special changelog with the operation of `IMPORT` and the timestamp of epoch. This ensures that any operation on an imported document supersedes the import record.

* Existing documents collection: What is the path of the the Cloud Firestore Collection you would like to import from? (This may, or may not, be the same Collection for which you plan to mirror changes.) If you want to use a collectionGroup query, provide the collection name value here, and set 'Use Collection Group query' to true.

* Use Collection Group query: Do you want to use a [collection group](https://firebase.google.com/docs/firestore/query-data/queries#collection-group-query) query for importing existing documents? Warning: A collectionGroup query will target every collection in your Firestore project that matches the 'Existing documents collection'. For example, if you have 10,000 documents with a sub-collection named: landmarks, this will query every document in 10,000 landmarks collections.

* Docs per backfill: When importing existing documents, how many should be imported at once? The default value of 200 should be ok for most users. If you are using a transform function or have very large documents, you may need to set this to a lower number. If the lifecycle event function times out, lower this value.

* Cloud KMS key name: Instead of Google managing the key encryption keys that protect your data, you control and manage key encryption keys in Cloud KMS. If this parameter is set, the extension will specify the KMS key name when creating the BQ table. See the PREINSTALL.md for more details.


Expand Down
2 changes: 1 addition & 1 deletion firestore-bigquery-export/extension.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.

name: firestore-bigquery-export
version: 0.1.41
version: 0.1.42
specVersion: v1beta

displayName: Stream Firestore to BigQuery
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
"url": "github.com/firebase/extensions.git",
"directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
},
"version": "1.1.29",
"version": "1.1.30",
"description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
"main": "./lib/index.js",
"scripts": {
"build": "npm run clean && npm run compile",
"clean": "rimraf lib",
"compile": "tsc",
"test:local": "firebase ext:dev:emulators:exec ./node_modules/.bin/jest --test-params=./src/__tests__/emulator-params.env --project=extensions-testing --config=./src/__tests__/firebase.json",
"test:local": "jest",
"prepare": "npm run build",
"generate-stresstest-table": "bq query --project_id=extensions-testing --use_legacy_sql=false < ./src/__tests__/fixtures/sql/generateSnapshotStresstestTable.sql"
},
Expand All @@ -35,16 +35,16 @@
"traverse": "^0.6.6"
},
"devDependencies": {
"@types/chai": "^4.1.6",
"@types/jest": "^24.0.18",
"@types/node": "14.18.34",
"@types/traverse": "^0.6.32",
"typescript": "^4.9.4",
"rimraf": "^2.6.3",
"nyc": "^14.0.0",
"jest": "^24.9.0",
"chai": "^4.2.0",
"ts-node": "^7.0.1",
"jest": "^24.9.0",
"nyc": "^14.0.0",
"rimraf": "^2.6.3",
"ts-jest": "^24.1.0",
"@types/jest": "^24.0.18",
"@types/chai": "^4.1.6"
"ts-node": "^7.0.1",
"typescript": "^4.9.4"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ describe("Using an alternative bigquery project", () => {
bqProjectId = "messaging-test-4395c";
});

test("successfully uses alternative project name when provided", async () => {
xtest("successfully uses alternative project name when provided", async () => {
const event: FirestoreDocumentChangeEvent = changeTrackerEvent({});

await changeTracker({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { ChangeType, FirestoreDocumentChangeEvent } from "../..";
import { FirestoreBigQueryEventHistoryTrackerConfig } from "../../bigquery";
import { Partitioning } from "../../bigquery/partitioning";
import { deleteTable } from "../fixtures/clearTables";
import { changeTracker } from "../fixtures/changeTracker";

let bq: BigQuery;
let dataset: Dataset;
Expand Down Expand Up @@ -197,6 +198,110 @@ describe("processing partitions on a new table", () => {
expect(value.end_date).toBeDefined();
});

test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined, with a timestamp-like value", async () => {
const config: FirestoreBigQueryEventHistoryTrackerConfig = {
datasetId: "",
tableId: "",
datasetLocation: "",
timePartitioning: "",
timePartitioningField: "end_date",
timePartitioningFieldType: "DATETIME",
timePartitioningFirestoreField: "end_date",
transformFunction: "",
clustering: [],
bqProjectId: null,
};

// a Timestamp-Like object (we lose the instance after serialization)
const end_date = {
_seconds: 1614153600,
_nanoseconds: 0,
};

const event: FirestoreDocumentChangeEvent = {
timestamp: "",
operation: ChangeType.CREATE,
documentName: "",
eventId: "",
documentId: "",
data: { end_date },
};

const partitioning = new Partitioning(config, table);
const value = partitioning.getPartitionValue(event);

expect(value.end_date).toBeDefined();
});

test("returns an empty object when _seconds or _nanoseconds is not a number", async () => {
const config: FirestoreBigQueryEventHistoryTrackerConfig = {
datasetId: "",
tableId: "",
datasetLocation: "",
timePartitioning: "",
timePartitioningField: "end_date",
timePartitioningFieldType: "DATETIME",
timePartitioningFirestoreField: "end_date",
transformFunction: "",
clustering: [],
bqProjectId: null,
};

// a Timestamp-Like object (we lose the instance after serialization)
const end_date = {
_seconds: "not a number",
_nanoseconds: 0,
};

const event: FirestoreDocumentChangeEvent = {
timestamp: "",
operation: ChangeType.CREATE,
documentName: "",
eventId: "",
documentId: "",
data: { end_date },
};

const partitioning = new Partitioning(config, table);
const value = partitioning.getPartitionValue(event);

expect(value).toEqual({});
});

test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined, and is timestamp-like", async () => {
const config: FirestoreBigQueryEventHistoryTrackerConfig = {
datasetId: "",
tableId: "",
datasetLocation: "",
timePartitioning: "",
timePartitioningField: "end_date",
timePartitioningFieldType: "DATETIME",
timePartitioningFirestoreField: "end_date",
transformFunction: "",
clustering: [],
bqProjectId: null,
};

// a Timestamp-Like object (we lose the instance after serialization)
const end_date = JSON.parse(
JSON.stringify(admin.firestore.Timestamp.now())
);

const event: FirestoreDocumentChangeEvent = {
timestamp: "",
operation: ChangeType.CREATE,
documentName: "",
eventId: "",
documentId: "",
data: { end_date },
};

const partitioning = new Partitioning(config, table);
const value = partitioning.getPartitionValue(event);

expect(value.end_date).toBeDefined();
});

test("returns an empty object if timePartitioningFirestoreField has not been provided", async () => {
const config: FirestoreBigQueryEventHistoryTrackerConfig = {
datasetId: "",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ let dataset: Dataset;
let table: Table;
let view: Table;
const count = 100;
describe("Stress testing", () => {
describe.skip("Stress testing", () => {
beforeEach(() => {
randomID = (Math.random() + 1).toString(36).substring(7);
datasetId = `dataset_${randomID}`;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ export class Partitioning {

private isValidPartitionTypeDate(value) {
/* Check if valid timestamp value from sdk */
if (value instanceof firebase.firestore.Timestamp) return true;
// if (value instanceof firebase.firestore.Timestamp) return true;
if (isTimestampLike(value)) return true;

/* Check if valid date/timstemap, expedted result from production */
if (value && value.toDate && value.toDate()) return true;
Expand Down Expand Up @@ -189,7 +190,7 @@ export class Partitioning {
Extracts a valid Partition field from the Document Change Event.
Matches result based on a pre-defined Firestore field matching the event data object.
Return an empty object if no field name or value provided.
Returns empty object if not a string or timestamp
Returns empty object if not a string or timestamp (or result of serializing a timestamp)
Logs warning if not a valid datatype
Delete changes events have no data, return early as cannot partition on empty data.
**/
Expand All @@ -210,6 +211,15 @@ export class Partitioning {

if (this.isValidPartitionTypeDate(fieldValue)) {
/* Return converted console value */
if (isTimestampLike(fieldValue)) {
const convertedTimestampFieldValue = convertToTimestamp(fieldValue);
return {
[fieldName]: this.convertDateValue(
convertedTimestampFieldValue.toDate()
),
};
}

if (fieldValue.toDate) {
return { [fieldName]: this.convertDateValue(fieldValue.toDate()) };
}
Expand Down Expand Up @@ -309,3 +319,27 @@ export class Partitioning {
}
}
}

type TimestampLike = {
_seconds: number;
_nanoseconds: number;
};

const isTimestampLike = (value: any): value is TimestampLike => {
if (value instanceof firebase.firestore.Timestamp) return true;
return (
typeof value === "object" &&
value !== null &&
"_seconds" in value &&
typeof value["_seconds"] === "number" &&
"_nanoseconds" in value &&
typeof value["_nanoseconds"] === "number"
);
};

const convertToTimestamp = (
value: TimestampLike
): firebase.firestore.Timestamp => {
if (value instanceof firebase.firestore.Timestamp) return value;
return new firebase.firestore.Timestamp(value._seconds, value._nanoseconds);
};
14 changes: 7 additions & 7 deletions firestore-bigquery-export/functions/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion firestore-bigquery-export/functions/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"author": "Jan Wyszynski <wyszynski@google.com>",
"license": "Apache-2.0",
"dependencies": {
"@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.29",
"@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.30",
"@google-cloud/bigquery": "^4.7.0",
"@types/chai": "^4.1.6",
"@types/express-serve-static-core": "4.17.30",
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 01d9102

Please sign in to comment.