Improvements and Docker

.dockerignore

@@ -0,0 +1,4 @@
+Dockerfile
+dist
+data
+_gen

.gitignore

@@ -33,3 +33,6 @@ __pycache__
 pyvenv.cfg
 .venv
 pip-selfcheck.json
+
+/data
+/_gen

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3-alpine
+
+RUN pip install poetry
+
+ADD . /ditto
+
+WORKDIR /ditto
+
+RUN poetry install
+CMD poetry run ditto clone --dest-dir ./data && \
+    poetry run ditto clone --src-url http://localhost/ --dest-dir ./data --select pokemon/129 && \
+    poetry run ditto analyze --data-dir ./data && \
+    poetry run ditto transform \
+        --base-url='https://pokeapi.co' \
+        --src-dir=./data \
+        --dest-dir=./_gen

README.md

@@ -14,3 +14,11 @@ This repository contains:
 pip install pokeapi-ditto
 ditto --help
 ```
+
+## Docker
+
+You should have a Pokeapi server running on `localhost:80`.
+
+```sh
+docker-compose up --build
+```

docker-compose.yml

@@ -0,0 +1,9 @@
+version: "2.3"
+
+services:
+  ditto:
+    build: .
+    volumes:
+      - ./data:/ditto/data
+      - ./_gen:/ditto/_gen
+    network_mode: "host"

pokeapi_ditto/commands/clone.py

@@ -11,7 +11,7 @@
 
 
 def _do_in_parallel(worker: Callable, data: List, desc: str) -> None:
-    cpus = os.cpu_count()
+    cpus = os.cpu_count() - 1
     pool = Pool(cpus, initializer=lambda: signal(SIGINT, SIG_IGN))
     try:
         for _ in tqdm(pool.imap_unordered(worker, data), total=len(data), desc=f"{desc} ({cpus}x)"):
@@ -65,10 +65,10 @@ def _crawl_resource_list(self, url: URL) -> List[URL]:
 
     def clone_single(self, endpoint_and_id: Tuple[str, str]) -> None:
         endpoint, id = endpoint_and_id
-        res_url = self._src_url / "api/v2" / endpoint / id
+        res_url = URL("{}/api/v2/{}/{}/".format(self._src_url, endpoint, id))
         self._crawl(res_url)
         if endpoint == "pokemon":
-            self._crawl(res_url / "encounters")
+            self._crawl(URL("{}encounters/".format(res_url)))
 
     def clone_endpoint(self, endpoint: str):
         res_list_url = self._src_url / "api/v2" / endpoint