Skip to content

Compression optimization #200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aramikm merged 3 commits into from
May 2, 2024
Merged

Compression optimization #200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4266220
bump max page id
aramikm May 1, 2024
8048d65
apply compression optimization
aramikm May 1, 2024
a366354
compression_optimization
aramikm May 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions bridge/common/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
[package]
name = "dsnp-graph-sdk-common"
version = "1.1.1"
version = "1.1.2"
edition = "2021"
license = "Apache-2.0"
publish = false

[dependencies]
dsnp-graph-core = { version = "1.1.1", path = "../../core" }
dsnp-graph-config = { version = "1.1.1", path = "../../config" }
dsnp-graph-core = { version = "1.1.2", path = "../../core" }
dsnp-graph-config = { version = "1.1.2", path = "../../config" }
libc = "0.2.153"
protobuf = { version = "3.4.0", features = ["with-bytes"] }
6 changes: 3 additions & 3 deletions bridge/ffi/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "dsnp-graph-sdk-ffi"
version = "1.1.1"
version = "1.1.2"
edition = "2021"
license = "Apache-2.0"
publish = false
Expand All @@ -10,8 +10,8 @@ name = "dsnp_graph_sdk_ffi"
crate-type = ["staticlib"]

[dependencies]
dsnp-graph-core = { version = "1.1.1", path = "../../core" }
dsnp-graph-config = { version = "1.1.1", path = "../../config" }
dsnp-graph-core = { version = "1.1.2", path = "../../core" }
dsnp-graph-config = { version = "1.1.2", path = "../../config" }
libc = "0.2.153"
lazy_static = "1.4.0"
anyhow = "1.0.80"
8 changes: 4 additions & 4 deletions bridge/jni/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "dsnp-graph-sdk-jni"
version = "1.1.1"
version = "1.1.2"
edition = "2021"
license = "Apache-2.0"
publish = false
Expand All @@ -10,9 +10,9 @@ name = "dsnp_graph_sdk_jni"
crate-type = ["cdylib"]

[dependencies]
dsnp-graph-core = { version = "1.1.1", path = "../../core" }
dsnp-graph-config = { version = "1.1.1", path = "../../config" }
dsnp-graph-sdk-common = { version = "1.1.1", path = "../common" }
dsnp-graph-core = { version = "1.1.2", path = "../../core" }
dsnp-graph-config = { version = "1.1.2", path = "../../config" }
dsnp-graph-sdk-common = { version = "1.1.2", path = "../common" }
jni = "0.21.1"
cfg-if = "1.0.0"
log = { version = "^0.4.21", features = ["std", "max_level_debug", "release_max_level_debug"] }
Expand Down
6 changes: 3 additions & 3 deletions bridge/node/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "dsnp-graph-sdk-node"
version = "1.1.1"
version = "1.1.2"
edition = "2021"
license = "Apache-2.0"
publish = false
Expand All @@ -11,8 +11,8 @@ name = "dsnp_graph_sdk_node"
crate-type = ["cdylib"]

[dependencies]
dsnp-graph-core = { version = "1.1.1", path = "../../core" }
dsnp-graph-config = { version = "1.1.1", path = "../../config" }
dsnp-graph-core = { version = "1.1.2", path = "../../core" }
dsnp-graph-config = { version = "1.1.2", path = "../../config" }
neon = { version = "1.0.0", default-features = false, features = ["napi-6"] }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.114"
Expand Down
4 changes: 2 additions & 2 deletions bridge/node/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions bridge/node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@dsnp/graph-sdk",
"version": "1.1.1",
"version": "1.1.2",
"author": "Amplica Labs",
"license": "Apache-2.0",
"description": "dsnp-graph-sdk-node",
Expand Down Expand Up @@ -55,6 +55,6 @@
},
"homepage": "https://github.com/LibertyDSNP/graph-sdk/bridge/node/README.md",
"customProperties": {
"uploadedBinariesVersion": "v1.1.1"
"uploadedBinariesVersion": "v1.1.2"
}
}
2 changes: 1 addition & 1 deletion config/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ homepage = "https://spec.dsnp.org"
license = "Apache-2.0"
publish = false
repository = "https://github.com/libertyDSNP/graph-sdk/"
version = "1.1.1"
version = "1.1.2"

[lib]
name = "dsnp_graph_config"
Expand Down
4 changes: 2 additions & 2 deletions core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ homepage = "https://spec.dsnp.org"
license = "Apache-2.0"
publish = false
repository = "https://github.com/libertyDSNP/graph-sdk/"
version = "1.1.1"
version = "1.1.2"

[lib]
name = "dsnp_graph_core"
Expand All @@ -16,7 +16,7 @@ doctest = false
anyhow = "1.0.69"
apache-avro = { version = "0.16.0", features = ["snappy"] }
dryoc = "0.5.3"
dsnp-graph-config = { version = "1.1.1", path = "../config" }
dsnp-graph-config = { version = "1.1.2", path = "../config" }
lazy_static = "1.4.0"
log = { version = "^0.4.21", features = ["std", "max_level_debug", "release_max_level_debug"] }
log-result-proc-macro = { path = "../log-result-proc-macro" }
Expand Down
8 changes: 7 additions & 1 deletion core/src/graph/graph.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,13 +256,15 @@ impl Graph {
})
.collect();

let ids_to_add: Vec<DsnpUserId> = updates
let mut ids_to_add: Vec<DsnpUserId> = updates
.iter()
.filter_map(|event| match event {
UpdateEvent::Add { dsnp_user_id, .. } => Some(*dsnp_user_id),
_ => None,
})
.collect();
// helps with the compression
ids_to_add.sort();

// First calculate pages that have had connections removed. Later, we will
// prefer to use these pages first to add new connections, so as to minimize
Expand Down Expand Up @@ -665,6 +667,10 @@ impl Graph {
.iter()
.filter(|c| !ids_to_add.contains(&c.user_id))
{
// This timestamp condition is eliminating PRID checks for 2 categories of connections
// 1. Connections that are just added but not included in `ids_to_add` list
// 2. Connections that are added less than `max_allowed_stale_days` since we need some time for the other
// side of the connection to act on it
if duration_days_since(c.since) > max_allowed_stale_days &&
!self
.user_key_manager
Expand Down