Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement threaded search in rest of modes(except empty folders) #504

Merged
merged 5 commits into from
Dec 17, 2021
Merged

Implement threaded search in rest of modes(except empty folders) #504

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ca68d86
Implement threaded search in rest of modes(except empty folders)
qarmin Dec 17, 2021
1ed9ed4
Add more tools in gui to use allowed extension
qarmin Dec 17, 2021
92a311b
Change progress window to modal
qarmin Dec 17, 2021
149865a
Nazair
qarmin Dec 17, 2021
764705e
Clippy
qarmin Dec 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 96 additions & 71 deletions czkawka_core/src/big_file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use std::collections::BTreeMap;
use std::ffi::OsStr;
use std::fs::{File, Metadata};
use std::io::{BufWriter, Write};
use std::path::PathBuf;
Expand All @@ -13,6 +12,7 @@ use std::{fs, thread};

use crossbeam_channel::Receiver;
use humansize::{file_size_opts as options, FileSize};
use rayon::prelude::*;

use crate::common::Common;
use crate::common_directory::Directories;
Expand Down Expand Up @@ -156,90 +156,116 @@ impl BigFile {
}

//// PROGRESS THREAD END

while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
// Be sure that every thread is closed
// End thread which send info to gui
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
return false;
}

let current_folder = folders_to_check.pop().unwrap();
let read_dir = match fs::read_dir(&current_folder) {
Ok(t) => t,
Err(e) => {
self.text_messages.warnings.push(format!("Cannot open dir {}, reason {}", current_folder.display(), e));
continue;
} // Permissions denied
};
'dir: for entry in read_dir {
let entry_data = match entry {
Ok(t) => t,
Err(e) => {
self.text_messages.warnings.push(format!("Cannot read entry in dir {}, reason {}", current_folder.display(), e));
continue;
} //Permissions denied
};
let metadata: Metadata = match entry_data.metadata() {
Ok(t) => t,
Err(e) => {
self.text_messages.warnings.push(format!("Cannot read metadata in dir {}, reason {}", current_folder.display(), e));
continue;
} //Permissions denied
};
if metadata.is_dir() {
if !self.recursive_search {
continue;
}
let segments: Vec<_> = folders_to_check
.par_iter()
.map(|current_folder| {
let mut dir_result = vec![];
let mut warnings = vec![];
let mut fe_result = vec![];
// Read current dir childrens
let read_dir = match fs::read_dir(&current_folder) {
Ok(t) => t,
Err(e) => {
warnings.push(format!("Cannot open dir {}, reason {}", current_folder.display(), e));
return (dir_result, warnings, fe_result);
}
};

let next_folder = current_folder.join(entry_data.file_name());
if self.directories.is_excluded(&next_folder) || self.excluded_items.is_excluded(&next_folder) {
continue 'dir;
}
// Check every sub folder/file/link etc.
'dir: for entry in read_dir {
let entry_data = match entry {
Ok(t) => t,
Err(e) => {
warnings.push(format!("Cannot read entry in dir {}, reason {}", current_folder.display(), e));
continue 'dir;
}
};
let metadata: Metadata = match entry_data.metadata() {
Ok(t) => t,
Err(e) => {
warnings.push(format!("Cannot read metadata in dir {}, reason {}", current_folder.display(), e));
continue 'dir;
}
};
if metadata.is_dir() {
if !self.recursive_search {
continue 'dir;
}

folders_to_check.push(next_folder);
} else if metadata.is_file() {
atomic_file_counter.fetch_add(1, Ordering::Relaxed);
// Extracting file extension
let file_extension = entry_data.path().extension().and_then(OsStr::to_str).map(str::to_lowercase);

// Checking allowed extensions
if !self.allowed_extensions.file_extensions.is_empty() {
let allowed = self.allowed_extensions.file_extensions.iter().map(|e| e.to_lowercase()).any(|e| file_extension == Some(e));
if !allowed {
// Not an allowed extension, ignore it.
continue 'dir;
}
}
let next_folder = current_folder.join(entry_data.file_name());
if self.directories.is_excluded(&next_folder) {
continue 'dir;
}

// Checking expressions
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
continue 'dir;
}
if self.excluded_items.is_excluded(&next_folder) {
continue 'dir;
}

// Creating new file entry
let fe: FileEntry = FileEntry {
path: current_file_name.clone(),
size: metadata.len(),
modified_date: match metadata.modified() {
Ok(t) => match t.duration_since(UNIX_EPOCH) {
Ok(d) => d.as_secs(),
dir_result.push(next_folder);
} else if metadata.is_file() {
atomic_file_counter.fetch_add(1, Ordering::Relaxed);

let file_name_lowercase: String = match entry_data.file_name().into_string() {
Ok(t) => t,
Err(_inspected) => {
self.text_messages.warnings.push(format!("File {} seems to be modified before Unix Epoch.", current_file_name.display()));
0
warnings.push(format!("File {:?} has not valid UTF-8 name", entry_data));
continue 'dir;
}
},
Err(e) => {
self.text_messages.warnings.push(format!("Unable to get modification date from file {}, reason {}", current_file_name.display(), e));
0
}
},
};
.to_lowercase();

self.big_files.entry(metadata.len()).or_insert_with(Vec::new);
self.big_files.get_mut(&metadata.len()).unwrap().push(fe);
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
continue 'dir;
}

let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
continue 'dir;
}

let fe: FileEntry = FileEntry {
path: current_file_name.clone(),
size: metadata.len(),
modified_date: match metadata.modified() {
Ok(t) => match t.duration_since(UNIX_EPOCH) {
Ok(d) => d.as_secs(),
Err(_inspected) => {
warnings.push(format!("File {} seems to be modified before Unix Epoch.", current_file_name.display()));
0
}
},
Err(e) => {
warnings.push(format!("Unable to get modification date from file {}, reason {}", current_file_name.display(), e));
0
}
},
};

fe_result.push((fe.size, fe));
}
}
(dir_result, warnings, fe_result)
})
.collect();

// Advance the frontier
folders_to_check.clear();

// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
for (size, fe) in fe_result {
self.big_files.entry(size).or_insert_with(Vec::new);
self.big_files.get_mut(&size).unwrap().push(fe);
}
}
}
Expand Down Expand Up @@ -345,7 +371,6 @@ impl DebugPrint for BigFile {

println!("### Other");
println!("Big files size {} in {} groups", self.information.number_of_real_files, self.big_files.len());
println!("Allowed extensions - {:?}", self.allowed_extensions.file_extensions);
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
Expand Down
Loading