Skip to content

Commit a83f4d6

Browse files
masx200masx200
committed
export module leetcode_test.web_crawler_multithreaded.Solution;
1 parent dd9f13d commit a83f4d6

File tree

9 files changed

+160
-9
lines changed

9 files changed

+160
-9
lines changed

‎construct-quad-tree/construct-quad-tree.vcxproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
<PlatformToolset>v143</PlatformToolset>
4141
</PropertyGroup>
4242
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
43-
<ConfigurationType>Application</ConfigurationType>
43+
<ConfigurationType>StaticLibrary</ConfigurationType>
4444
<UseDebugLibraries>true</UseDebugLibraries>
4545
<PlatformToolset>v143</PlatformToolset>
4646
</PropertyGroup>

‎construct-quad-tree/construct-quad-tree.vcxproj.filters

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@
1515
</Filter>
1616
</ItemGroup>
1717
<ItemGroup>
18-
<ClCompile Include="index.ixx" />
19-
<ClCompile Include="Node.ixx" />
18+
<ClCompile Include="index.ixx">
19+
<Filter>Source Files</Filter>
20+
</ClCompile>
21+
<ClCompile Include="Node.ixx">
22+
<Filter>Source Files</Filter>
23+
</ClCompile>
2024
</ItemGroup>
2125
</Project>

‎web-crawler-multithreaded/CMakeLists.txt

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# this is the build file for project
2+
# it is autogenerated by the xmake build system.
3+
# do not edit by hand.
4+
5+
# project
6+
cmake_minimum_required(VERSION 3.15.0)
7+
cmake_policy(SET CMP0091 NEW)
8+
project(web-crawler-multithreaded LANGUAGES CXX C)
9+
10+
# target
11+
add_library(web-crawler-multithreaded STATIC "" index.ixx HtmlParser.ixx)
12+
set_target_properties(web-crawler-multithreaded PROPERTIES OUTPUT_NAME "web-crawler-multithreaded")
13+
set_target_properties(web-crawler-multithreaded PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}/build/windows/x64/release")
14+
add_custom_command(OUTPUT output_web-crawler-multithreaded_66E70B65
15+
COMMAND echo [ 0%]: compiling.module.release leetcode_test.web_crawler_multithreaded.HtmlParser
16+
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/build/.objs/web-crawler-multithreaded/windows/x64/release
17+
COMMAND "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.36.32532/bin/HostX64/x64/cl.exe" -nologo -std:c++20 /EHsc -experimental:module -TP -c ${CMAKE_SOURCE_DIR}/HtmlParser.ixx -Fo${CMAKE_SOURCE_DIR}/build/.objs/web-crawler-multithreaded/windows/x64/release/HtmlParser.ixx.obj -ifcOutput ${CMAKE_SOURCE_DIR}/build/.gens/web-crawler-multithreaded/windows/x64/release/rules/modules/cache/df3274b1/leetcode_test.web_crawler_multithreaded.HtmlParser.ifc -interface
18+
COMMAND echo [ 0%]: compiling.module.release leetcode_test.web_crawler_multithreaded.Solution
19+
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/build/.objs/web-crawler-multithreaded/windows/x64/release
20+
COMMAND "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.36.32532/bin/HostX64/x64/cl.exe" -nologo -std:c++20 /EHsc -experimental:module -TP -c ${CMAKE_SOURCE_DIR}/index.ixx -Fo${CMAKE_SOURCE_DIR}/build/.objs/web-crawler-multithreaded/windows/x64/release/index.ixx.obj -ifcOutput ${CMAKE_SOURCE_DIR}/build/.gens/web-crawler-multithreaded/windows/x64/release/rules/modules/cache/df3274b1/leetcode_test.web_crawler_multithreaded.Solution.ifc -interface -reference leetcode_test.web_crawler_multithreaded.HtmlParser=${CMAKE_SOURCE_DIR}/build/.gens/web-crawler-multithreaded/windows/x64/release/rules/modules/cache/df3274b1/leetcode_test.web_crawler_multithreaded.HtmlParser.ifc
21+
VERBATIM
22+
)
23+
add_custom_target(target_web-crawler-multithreaded_66E70B65
24+
DEPENDS output_web-crawler-multithreaded_66E70B65
25+
)
26+
add_dependencies(web-crawler-multithreaded target_web-crawler-multithreaded_66E70B65)
27+
target_compile_options(web-crawler-multithreaded PRIVATE
28+
$<$<COMPILE_LANGUAGE:CXX>:-experimental:module>
29+
)
30+
if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
31+
target_compile_options(web-crawler-multithreaded PRIVATE /EHsc)
32+
else()
33+
target_compile_options(web-crawler-multithreaded PRIVATE -fcxx-exceptions)
34+
endif()
35+
set_target_properties(web-crawler-multithreaded PROPERTIES CXX_EXTENSIONS OFF)
36+
target_compile_features(web-crawler-multithreaded PRIVATE cxx_std_20)
37+
if(MSVC)
38+
set_property(TARGET web-crawler-multithreaded PROPERTY
39+
MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
40+
endif()
41+
target_link_libraries(web-crawler-multithreaded PRIVATE
42+
${CMAKE_SOURCE_DIR}/build/.objs/web-crawler-multithreaded/windows/x64/release/HtmlParser.ixx.obj
43+
${CMAKE_SOURCE_DIR}/build/.objs/web-crawler-multithreaded/windows/x64/release/index.ixx.obj
44+
)
45+
target_sources(web-crawler-multithreaded PRIVATE
46+
)
47+

‎web-crawler-multithreaded/HtmlParser.ixx

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,32 @@ module;
22
#include <chrono>
33
#include <string>
44
#include <thread>
5+
#include <unordered_map>
6+
#include <utility>
57
#include <vector>
68
export module leetcode_test.web_crawler_multithreaded.HtmlParser;
79
using std::string;
810
using std::vector;
11+
using namespace std;
912
namespace leetcode_test::web_crawler_multithreaded {
1013
export class HtmlParser {
14+
private:
15+
unordered_map<string, vector<string>> urlMap;
1116

1217
public:
13-
vector<string> getUrls(string url)
18+
vector<string> getUrls(string& url)
1419
{
1520
using namespace std::chrono_literals;
1621
std::this_thread::sleep_for(15ms);
22+
return urlMap[url];
1723
}
1824

19-
HtmlParser(vector<string>& urls, vector<vector<int>>& edges) { }
25+
HtmlParser(vector<string>& urls, vector<pair<int, int>>& edges)
26+
{
27+
for (auto& [k, v] : edges) {
28+
urlMap[urls[k]].emplace_back(urls[v]);
29+
}
30+
}
2031
};
2132

2233
} // namespace leetcode_test::web_crawler_multithreaded

‎web-crawler-multithreaded/format.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"style": "WebKit",
3+
"filter": [
4+
"*.ixx",
5+
"*.c",
6+
"*.cc",
7+
"*.cpp",
8+
"*.h",
9+
"*.hh",
10+
"*.hpp"
11+
],
12+
"dirs": [
13+
"./"
14+
]
15+
}

‎web-crawler-multithreaded/index.ixx

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,80 @@
11
module;
2+
#include <future>
23
#include <string>
4+
#include <thread>
5+
#include <unordered_set>
36
#include <vector>
47
export module leetcode_test.web_crawler_multithreaded.Solution;
58
import leetcode_test.web_crawler_multithreaded.HtmlParser;
9+
using namespace std;
10+
611
using std::string;
712
using std::vector;
13+
814
namespace leetcode_test::web_crawler_multithreaded {
915

1016
export class Solution {
17+
private:
18+
unordered_set<string> seen;
19+
1120
public:
12-
vector<string> crawl(string startUrl, HtmlParser htmlParser) { }
21+
vector<string> crawl(string& startUrl, HtmlParser& htmlParser)
22+
{
23+
seen.insert(startUrl);
24+
auto origin = getHostname(startUrl);
25+
auto urls = getUrls(startUrl, htmlParser).get();
26+
auto sameOriginUrls = vector<string> {};
27+
for (auto& url : urls) {
28+
if (getHostname(url) == origin && !seen.contains(url)) {
29+
sameOriginUrls.emplace_back(url);
30+
seen.insert(url);
31+
}
32+
}
33+
if (sameOriginUrls.size()) {
34+
crawlMany(sameOriginUrls, htmlParser, origin);
35+
}
36+
return { seen.begin(), seen.end() };
37+
}
1338

1439
private:
40+
void crawlMany(vector<string>& urls, HtmlParser& htmlParser, string& origin)
41+
{
42+
if (urls.empty()) {
43+
return;
44+
}
45+
auto resUrls = vector<string> {};
46+
auto futures = vector<future<vector<string>>> {};
47+
for (auto& url : urls) {
48+
futures.emplace_back(getUrls(url, htmlParser));
49+
}
50+
for (auto& f : futures) {
51+
52+
for (auto& u : f.get()) {
53+
resUrls.emplace_back(u);
54+
}
55+
}
56+
auto sameOriginUrls = vector<string> {};
57+
for (auto& url : resUrls) {
58+
if (getHostname(url) == origin && !seen.contains(url)) {
59+
sameOriginUrls.emplace_back(url);
60+
seen.insert(url);
61+
}
62+
}
63+
if (sameOriginUrls.size()) {
64+
crawlMany(sameOriginUrls, htmlParser, origin);
65+
}
66+
}
67+
future<vector<string>> getUrls(string& startUrl, HtmlParser& htmlParser)
68+
{
69+
70+
return async([&]() {
71+
return htmlParser.getUrls(startUrl);
72+
});
73+
}
1574
static string getHostname(const string& url)
1675
{
17-
const int firstSlash = url.find_first_of('/');
18-
const int thirdSlash = url.find_first_of('/', firstSlash + 2);
76+
const auto firstSlash = url.find_first_of('/');
77+
const auto thirdSlash = url.find_first_of('/', firstSlash + 2);
1978
return url.substr(firstSlash + 2, thirdSlash - firstSlash - 2);
2079
}
2180
};

‎web-crawler-multithreaded/web-crawler-multithreaded.vcxproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
<PlatformToolset>v143</PlatformToolset>
3737
</PropertyGroup>
3838
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
39-
<ConfigurationType>Application</ConfigurationType>
39+
<ConfigurationType>StaticLibrary</ConfigurationType>
4040
<UseDebugLibraries>true</UseDebugLibraries>
4141
<PlatformToolset>v143</PlatformToolset>
4242
</PropertyGroup>

‎web-crawler-multithreaded/web-crawler-multithreaded.vcxproj.filters

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,12 @@
1414
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav</Extensions>
1515
</Filter>
1616
</ItemGroup>
17+
<ItemGroup>
18+
<ClCompile Include="HtmlParser.ixx">
19+
<Filter>Source Files</Filter>
20+
</ClCompile>
21+
<ClCompile Include="index.ixx">
22+
<Filter>Source Files</Filter>
23+
</ClCompile>
24+
</ItemGroup>
1725
</Project>

‎web-crawler-multithreaded/xmake.lua

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
set_languages('c++20')
2+
target("web-crawler-multithreaded")
3+
set_kind("static")
4+
5+
6+
add_files("*.ixx", {install = true})
7+
target_end()

0 commit comments

Comments
 (0)