Merge branch 'master' into release/0.1

Author: tublitzed

README.md

@@ -32,6 +32,10 @@ and can be run against a local server, e.g.:
         $ export SYNC_DATABASE_URL=mysql://<YOUR_MYSQL_USERNAME>:<YOUR_MYSQL_PASSWORD>@localhost/syncstorage
         $ cargo run
 
+## Logging
+
+If you want to connect to the existing [Sentry project](https://sentry.prod.mozaws.net/operations/syncstorage-dev/) for local development, login to Sentry, and go to the page with [api keys](https://sentry.prod.mozaws.net/settings/operations/syncstorage-dev/keys/). Copy the `DSN` value, and `export SENTRY_DSN=DSN_VALUE_GOES_HERE` to the environment when running this project.
+
 ## Running the Unit tests
 
 1) Run:

db-tests/Cargo.lock

@@ -6,9 +6,10 @@ edition = "2018"
 
 [dependencies]
 codegen = { version = "0.1.0", path = "../codegen" }
-env_logger = "0.6.1"
+env_logger = "0.7.0"
 futures-preview = { git = "https://github.com/rust-lang-nursery/futures-rs", rev = "744ece9", features = ["compat"] }
-lazy_static = "1.3.0"
-rand = "0.6.5"
+lazy_static = "1.4.0"
+log="0.4"
+rand = "0.7.2"
 syncstorage = { version = "0.1.0", path = "../", features = ["db_test"] }
 

db-tests/Cargo.toml

@@ -1,6 +1,8 @@
 use futures::compat::Future01CompatExt;
 
 use codegen::async_test;
+use log::debug;
+
 use syncstorage::{
     db::{error::DbErrorKind, params, util::SyncTimestamp, BATCH_LIFETIME},
     error::ApiErrorKind,
@@ -141,7 +143,7 @@ async fn append_commit() -> Result<()> {
         .compat()
         .await?;
 
-    dbg!("result", &result);
+    debug!("result: {:?}", &result);
     assert!(result.success.contains(&"b0".to_owned()));
     assert!(result.success.contains(&"b2".to_owned()));
 

db-tests/src/batch.rs

@@ -1,21 +1,29 @@
+-- fxa_uid: a 16 byte identifier, randomly generated by the fxa server
+--    usually a UUID, so presuming a formatted form.
+-- fxa_kid: <`mono_num`>-<`client_state`>
+--
+-- - mono_num: a monotonically increasing timestamp or generation number 
+--             in hex and padded to 13 digits, provided by the fxa server
+-- - client_state: the first 16 bytes of a SHA256 hash of the user's sync
+--             encryption key.
+--
+-- NOTE: DO NOT INCLUDE COMMENTS IF PASTING INTO CONSOLE
+--       ALSO, CONSOLE WANTS ONE SPACE BETWEEN DDL COMMANDS
+
 CREATE TABLE user_collections (
-  fxa_uid STRING(MAX)  NOT NULL,
-  fxa_kid STRING(MAX)  NOT NULL,
+  fxa_uid STRING(36)  NOT NULL,
+  fxa_kid STRING(48)  NOT NULL,
   collection_id INT64  NOT NULL,
   modified TIMESTAMP   NOT NULL,
 ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id);
 
-
 CREATE TABLE bso (
-  fxa_uid STRING(MAX)  NOT NULL,
-  fxa_kid STRING(MAX)  NOT NULL,
+  fxa_uid STRING(36)  NOT NULL,
+  fxa_kid STRING(48)  NOT NULL,
   collection_id INT64  NOT NULL,
-  id STRING(MAX)       NOT NULL,
-
+  id STRING(64)       NOT NULL,
   sortindex INT64,
-
   payload STRING(MAX)  NOT NULL,
-
   modified TIMESTAMP   NOT NULL,
   expiry TIMESTAMP     NOT NULL,
 )    PRIMARY KEY(fxa_uid, fxa_kid, collection_id, id),
@@ -28,19 +36,17 @@ INTERLEAVE IN user_collections;
     CREATE INDEX BsoExpiry
            ON bso(expiry);
 
-
 CREATE TABLE collections (
   id INT64          NOT NULL,
-  name STRING(MAX)  NOT NULL,
+  name STRING(32)  NOT NULL,
 ) PRIMARY KEY(id);
 
     CREATE UNIQUE INDEX CollectionName
         ON collections(name);
 
-
 CREATE TABLE batches (
-  fxa_uid STRING(MAX)  NOT NULL,
-  fxa_kid STRING(MAX)  NOT NULL,
+  fxa_uid STRING(36)  NOT NULL,
+  fxa_kid STRING(48)  NOT NULL,
   id TIMESTAMP         NOT NULL,
   collection_id INT64  NOT NULL,
   bsos STRING(MAX)     NOT NULL,

spanner-2019-10-01.ddl

@@ -2,6 +2,7 @@ use futures::future;
 use futures::lazy;
 
 use diesel::r2d2::PooledConnection;
+use log::debug;
 
 use std::cell::RefCell;
 use std::collections::HashMap;
@@ -439,7 +440,10 @@ impl SpannerDb {
         &self,
         params: params::GetCollectionTimestamp,
     ) -> Result<SyncTimestamp> {
-        dbg!("!!QQQ get_collection_timestamp_sync", &params.collection);
+        debug!(
+            "!!QQQ get_collection_timestamp_sync {:?}",
+            &params.collection
+        );
 
         let collection_id = self.get_collection_id(&params.collection)?;
         if let Some(modified) = self
@@ -1005,7 +1009,7 @@ impl SpannerDb {
     }
 
     pub fn get_bso_timestamp_sync(&self, params: params::GetBsoTimestamp) -> Result<SyncTimestamp> {
-        dbg!("!!QQQ get_bso_timestamp_sync", &params.collection);
+        debug!("!!QQQ get_bso_timestamp_sync: {:?}", &params.collection);
         let collection_id = self.get_collection_id(&params.collection)?;
 
         let result = self
@@ -1099,7 +1103,7 @@ impl SpannerDb {
         }
 
         if !inserts.is_empty() {
-            dbg!(&inserts);
+            debug!("inserts: {:?}", &inserts);
             self.insert(
                 "bso",
                 &[
@@ -1116,7 +1120,7 @@ impl SpannerDb {
             );
         }
         for (columns, values) in updates {
-            dbg!(&columns, &values);
+            debug!("columns: {:?}, values:{:?}", &columns, &values);
             self.update("bso", &columns, values);
         }
 
@@ -1266,7 +1270,10 @@ impl SpannerDb {
                 .map_or(i64::from(DEFAULT_BSO_TTL), |ttl| ttl.try_into().unwrap())
                 * 1000;
             let expirystring = to_rfc3339(now_millis + ttl)?;
-            dbg!("!!!!! INSERT", &expirystring, timestamp, ttl);
+            debug!(
+                "!!!!! INSERT expirystring:{:?}, timestamp:{:?}, ttl:{:?}",
+                &expirystring, timestamp, ttl
+            );
             sqlparams.insert("expiry".to_string(), as_value(expirystring));
             sqltypes.insert("expiry".to_string(), SpannerType::Timestamp.into());
 

src/db/spanner/models.rs

@@ -2,7 +2,7 @@
 use std::error::Error;
 
 use docopt::Docopt;
-use log::info;
+use log::{debug, info};
 use serde_derive::Deserialize;
 
 use syncstorage::{logging, server, settings};
@@ -22,6 +22,7 @@ struct Args {
 
 fn main() -> Result<(), Box<dyn Error>> {
     env_logger::init();
+    debug!("Starting up...");
     // Set SENTRY_DSN environment variable to enable Sentry
     let sentry = sentry::init(sentry::ClientOptions::default());
     if sentry.is_enabled() {

src/main.rs

@@ -2,6 +2,7 @@ use std::net::UdpSocket;
 
 use actix_web::{error::ErrorInternalServerError, Error, HttpRequest};
 use cadence::{BufferedUdpMetricSink, Counted, NopMetricSink, QueuingMetricSink, StatsdClient};
+use log::debug;
 
 use crate::error::ApiError;
 use crate::server::ServerState;
@@ -18,7 +19,7 @@ impl From<&HttpRequest> for Metrics {
             client: match req.app_data::<ServerState>() {
                 Some(v) => Some(*v.metrics.clone()),
                 None => {
-                    dbg!("⚠️ metric error: No App State");
+                    debug!("⚠️ metric error: No App State");
                     None
                 }
             },
@@ -45,7 +46,7 @@ impl Metrics {
             match self.client.unwrap().incr(label) {
                 Err(e) => {
                     // eat the metric, but log the error
-                    dbg!("⚠️ Metric {} error: {:?} ", label, e);
+                    debug!("⚠️ Metric {} error: {:?} ", label, e);
                 }
                 Ok(_v) => {
                     // v.as_metric_str()
@@ -79,7 +80,7 @@ pub fn metrics_from_opts(opts: &Settings) -> Result<StatsdClient, ApiError> {
     };
     Ok(builder
         .with_error_handler(|err| {
-            dbg!("⚠️ Metric send error:", err);
+            debug!("⚠️ Metric send error:  {:?}", err);
         })
         .build())
 }

src/server/metrics.rs

@@ -1,6 +1,7 @@
 //! Application settings objects and initialization
 
 use config::{Config, ConfigError, Environment, File};
+use log::debug;
 use serde::{de::Deserializer, Deserialize, Serialize};
 use url::Url;
 
@@ -118,11 +119,11 @@ impl Settings {
                         "For example to set `database_url` use env var `{}_DATABASE_URL`\n",
                         PREFIX.to_uppercase()
                     );
-                    dbg!("⚠️ Configuration error: Value undefined", &v);
+                    debug!("⚠️ Configuration error: Value undefined {:?}", &v);
                     return Err(ConfigError::NotFound(v));
                 }
                 _ => {
-                    dbg!("⚠️ Other: ", &e);
+                    debug!("⚠️ Other:  {:?}", &e);
                     return Err(e);
                 }
             },

src/settings.rs

@@ -16,6 +16,7 @@ use actix_web::{
 };
 use futures::{future, Future};
 use lazy_static::lazy_static;
+use log::debug;
 use regex::Regex;
 use serde::{
     de::{Deserializer, Error as SerdeError, IgnoredAny},
@@ -120,7 +121,7 @@ impl FromRequest for BsoBodies {
         let default = HeaderValue::from_static("");
         let content_type = get_trimmed_header(headers, CONTENT_TYPE, &default);
 
-        dbg!(&content_type);
+        debug!("content_type: {:?}", &content_type);
 
         match content_type.as_str() {
             "application/json" | "text/plain" | "application/newlines" | "" => (),
@@ -138,7 +139,7 @@ impl FromRequest for BsoBodies {
 
         // Load the entire request into a String
         let fut = <String>::from_request(req, payload).map_err(|e| {
-            dbg!("⚠️ Payload read error", e);
+            debug!("⚠️ Payload read error: {:?}", e);
             ValidationErrorKind::FromDetails(
                 "Mimetype/encoding/content-length error".to_owned(),
                 RequestErrorLocation::Header,
@@ -165,7 +166,7 @@ impl FromRequest for BsoBodies {
         let state = match req.app_data::<ServerState>() {
             Some(s) => s,
             None => {
-                dbg!("⚠️ Could not load the app state");
+                debug!("⚠️ Could not load the app state");
                 return Box::new(future::err(
                     ValidationErrorKind::FromDetails(
                         "Internal error".to_owned(),
@@ -315,7 +316,7 @@ impl FromRequest for BsoBody {
         let state = match req.app_data::<ServerState>() {
             Some(s) => s,
             None => {
-                dbg!("⚠️ Could not load the app state");
+                debug!("⚠️ Could not load the app state");
                 return Box::new(future::err(
                     ValidationErrorKind::FromDetails(
                         "Internal error".to_owned(),
@@ -391,7 +392,7 @@ impl BsoParam {
         }
         if let Some(v) = elements.get(5) {
             let sv = String::from_str(v).map_err(|e| {
-                dbg!("⚠️ BsoParam Error", v, e);
+                debug!("⚠️ BsoParam Error element:{:?} error:{:?}", v, e);
                 ValidationErrorKind::FromDetails(
                     "Invalid BSO".to_owned(),
                     RequestErrorLocation::Path,
@@ -615,7 +616,7 @@ impl FromRequest for CollectionPostRequest {
         let state = match req.app_data::<ServerState>() {
             Some(s) => s,
             None => {
-                dbg!("⚠️ Could not load the app state");
+                debug!("⚠️ Could not load the app state");
                 return Box::new(future::err(
                     ValidationErrorKind::FromDetails(
                         "Internal error".to_owned(),
@@ -798,7 +799,7 @@ impl FromRequest for ConfigRequest {
         let state = match req.app_data::<ServerState>() {
             Some(s) => s,
             None => {
-                dbg!("⚠️ Could not load the app state");
+                debug!("⚠️ Could not load the app state");
                 return Err(ValidationErrorKind::FromDetails(
                     "Internal error".to_owned(),
                     RequestErrorLocation::Unknown,
@@ -859,7 +860,7 @@ impl HawkIdentifier {
         let elements: Vec<&str> = uri.path().split('/').collect();
         if let Some(v) = elements.get(2) {
             u64::from_str(v).map_err(|e| {
-                dbg!("⚠️ HawkIdentifier Error", v, e);
+                debug!("⚠️ HawkIdentifier Error {:?} {:?}", v, e);
                 ValidationErrorKind::FromDetails(
                     "Invalid UID".to_owned(),
                     RequestErrorLocation::Path,
@@ -936,7 +937,7 @@ impl FromRequest for HawkIdentifier {
         let state = match req.app_data::<ServerState>() {
             Some(s) => s,
             None => {
-                dbg!("⚠️ Could not load the app state");
+                debug!("⚠️ Could not load the app state");
                 return Err(ValidationErrorKind::FromDetails(
                     "Internal error".to_owned(),
                     RequestErrorLocation::Unknown,
@@ -964,7 +965,7 @@ impl From<u32> for HawkIdentifier {
 
 pub fn extrude_db(exts: &Extensions) -> Result<Box<dyn Db>, Error> {
     exts.get::<Box<dyn Db>>().cloned().ok_or_else(|| {
-        dbg!("⚠️ DB Error: No db");
+        debug!("⚠️ DB Error: No db");
         ErrorInternalServerError("Unexpected Db error: No DB".to_owned())
     })
 }
@@ -1074,7 +1075,7 @@ impl FromRequest for BatchRequestOpt {
         let state = match req.app_data::<ServerState>() {
             Some(s) => s,
             None => {
-                dbg!("⚠️ Could not load the app state");
+                debug!("⚠️ Could not load the app state");
                 return Err(ValidationErrorKind::FromDetails(
                     "Internal error".to_owned(),
                     RequestErrorLocation::Unknown,