BE: use the same processing path converting from vfs to salsa

Summary:
We have a number of places that take the contents of a file from vfs and store them in the salsa database.
This processing includes converting from `[u8]` to a string in a safe way, and normalising line endings.
Put the processing in one place, and use it everywhere.

Note that the line_endings information is only used when generating output again, as in assists with edits. So it is ignored in the shell and CLI cases.

A later diff will make use of it for `elp lint`.

Reviewed By: robertoaloi

Differential Revision: D56474440

fbshipit-source-id: a7e82dd6a64e94fa07d840fcf7baebfb1f07ae29

crates/elp/src/bin/shell.rs

@@ -226,8 +226,8 @@ fn process_changes_to_vfs_store(loaded: &mut LoadResult) -> bool {
     for file in &changed_files {
         if file.exists() {
             let bytes = loaded.vfs.file_contents(file.file_id).to_vec();
-            let document = Document::from_bytes(bytes);
-            raw_database.set_file_text(file.file_id, Arc::from(document.content));
+            let (text, _line_ending) = Document::vfs_to_salsa(&bytes);
+            raw_database.set_file_text(file.file_id, Arc::from(text));
         } else {
             raw_database.set_file_text(file.file_id, Arc::from(""));
         };

crates/elp/src/build/load.rs

@@ -39,6 +39,7 @@ use elp_project_model::ProjectManifest;
 
 use crate::build::types::LoadResult;
 use crate::cli::Cli;
+use crate::document::Document;
 use crate::reload::ProjectFolders;
 
 pub fn load_project_at(
@@ -182,20 +183,9 @@ fn load_database(
     let changes = vfs.take_changes();
     for file in changes {
         if file.exists() {
-            let contents = vfs.file_contents(file.file_id).to_vec();
-            match String::from_utf8(contents) {
-                Ok(text) => {
-                    db.set_file_text(file.file_id, Arc::from(text));
-                }
-                Err(err) => {
-                    // Fall back to lossy latin1 loading of files.
-                    // This should only affect files from yaws, and
-                    // possibly OTP that are latin1 encoded.
-                    let contents = err.into_bytes();
-                    let text: String = contents.into_iter().map(|byte| byte as char).collect();
-                    db.set_file_text(file.file_id, Arc::from(text));
-                }
-            }
+            let bytes = vfs.file_contents(file.file_id).to_vec();
+            let (text, _line_ending) = Document::vfs_to_salsa(&bytes);
+            db.set_file_text(file.file_id, Arc::from(text));
             bump_file_revision(file.file_id, db);
         }
     }

crates/elp/src/document.rs

@@ -13,12 +13,19 @@ use elp_ide::elp_ide_db::LineIndex;
 use lsp_types::TextDocumentContentChangeEvent;
 
 use crate::from_proto::text_range;
+use crate::line_endings::LineEndings;
 
 pub struct Document {
     pub content: String,
 }
 
 impl Document {
+    pub fn vfs_to_salsa(vfs_bytes: &[u8]) -> (String, LineEndings) {
+        let bytes = vfs_bytes.to_vec();
+        let document = Document::from_bytes(bytes);
+        LineEndings::normalize(document.content)
+    }
+
     pub fn from_bytes(bytes: Vec<u8>) -> Document {
         let content = match String::from_utf8(bytes) {
             Ok(text) => text,

crates/elp/src/lib.rs

@@ -35,7 +35,7 @@ pub mod convert;
 pub mod document;
 mod from_proto;
 mod handlers;
-mod line_endings;
+pub mod line_endings;
 pub mod lsp_ext;
 mod op_queue;
 mod project_loader;

crates/elp/src/server.rs

@@ -861,12 +861,12 @@ impl Server {
                     &file.file_id, &file.change_kind
                 ));
                 let bytes = vfs.file_contents(file.file_id).to_vec();
-                let document = Document::from_bytes(bytes);
-                let (text, line_ending) = LineEndings::normalize(document.content);
+                let (text, line_ending) = Document::vfs_to_salsa(&bytes);
+                raw_database.set_file_text(file.file_id, Arc::from(text));
                 self.line_ending_map
                     .write()
                     .insert(file.file_id, line_ending);
-                raw_database.set_file_text(file.file_id, Arc::from(text));
+
                 // causes us to remove stale squiggles from the UI
                 Arc::make_mut(&mut self.diagnostics).set_eqwalizer(file.file_id, vec![]);
             } else {