Redesign base url handling

.gitignore

@@ -1,3 +1,5 @@
+data-dist/
+
 .idea/
 
 *.py[cod]

.stickler.yml

@@ -5,13 +5,13 @@ linters:
     fixer: true
 
   flake8:
-    python: 3
     fixer: true
-
-    # Make sure to copy changes to tox.ini too
+    python: 3
+
+    # copied from tox.ini
     max-complexity: 15
     ignore: W,E
-    exclude: .venv, .git, __pycache__, dist
+    exclude: .venv, .git, __pycache__, dist, data
 
 fixers:
   enable: true

Dockerfile

@@ -2,13 +2,18 @@ FROM python:3.6
 
 MAINTAINER Sargun Vohra <sargun.vohra@gmail.com>
 
-RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+ENV PYTHONUNBUFFERED 1
+ENV DITTO_BASE_URL http://localhost/
+
+EXPOSE 80
 
 RUN mkdir /ditto
 WORKDIR /ditto/
-ADD . /ditto/
 
-RUN poetry install
+COPY pyproject.* /ditto/
+COPY pokeapi_ditto /ditto/pokeapi_ditto
+COPY data /ditto/data
 
-CMD poetry run ditto serve
-EXPOSE 80
+RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+RUN poetry install
+CMD poetry run ditto serve "--base-url=$DITTO_BASE_URL"

README.md

@@ -4,46 +4,52 @@
 
 This repository contains:
 
- -   a static copy of the JSON data generated from
-     [PokeAPI](https://github.com/PokeAPI/pokeapi) based on
-     [Veekun’s data](https://github.com/veekun/pokedex)
- -   a PokeAPI schema generated from the above data
- -   a script to serve the data in the same form as PokeAPI
- -   a script to crawl an instance of PokeAPI to regenerate the data
- -   a script to analyze the generated data and produce a JSON Schema
+ - `ditto clone`: a script to crawl an instance of PokeAPI and download all objects
+ - `data/api`: a static copy of the JSON data generated with the above script
+ - `ditto analyze`: a script to generate a JSON schema of the above data
+ - `data/schema`: a static copy of the PokeAPI schema generated from the above data
+ - `ditto transform`: a script to apply a new base url to data in `data/api` 
+ - `ditto serve`: a script to serve the data in the same form as PokeAPI
+    - with full support for dynamic pagination using GET args `offset` and `limit`
 
-## Usage
+## Docker
+
+This project is on Docker Hub. If you just want to serve a PokeApi clone, you
+just have to run one command.
 
-This project is on Docker Hub. If you just want to run it, you just have
-to run one command. Replace `8080` with the port of your choice.
+ - Replace `8080` with the port of your choice
+ - Replace `http://localhost:8080` with the base url of your choice
 
 ``` bash
-docker run -p 8080:80 sargunv/pokeapi-ditto
+docker run -p 8080:80 -e DITTO_BASE_URL=http://localhost:8080 sargunv/pokeapi-ditto
 ```
 
-## Development
+## Usage
 
-If you plan to edit the project, you can install it locally for
-development. [Poetry](https://poetry.eustace.io/) is required.
+If you'd rather use the data for something else, you can generate a
+copy with the base url of your choice applied. This assumes
+[Poetry](https://poetry.eustace.io/) is installed and in your PATH. 
 
 ``` bash
-cd ~
 git clone https://github.com/PokeAPI/ditto.git
 cd ditto
 poetry install
-
-# now you can run ditto!
-poetry run ditto --help
+poetry run ditto transform --base-url http://localhost:8080
 ```
 
+For other ditto functionality, run `poetry run ditto --help` 
+
+If you're on Windows, you'll have to adapt the commands above to your platform.
+The general idea is the same.
+
 ## Advanced
 
 You can manually update the data if necessary. If I abandon this
 project, here’s how to update it. It's a bit of an involved process.
 
 Before starting, you’ll need to install [Docker and Docker
 Compose](https://docs.docker.com/compose/install/). You'll
-also need [Poetry](https://poetry.eustace.io/).
+also need [Poetry](https://poetry.eustace.io/) in your PATH.
 
 First clone the PokeAPI and Ditto repositories:
 
@@ -81,21 +87,21 @@ from data.v2.build import build_all
 build_all()
 ```
 
-The above step can take a really long time to complete. Once it’s done,
-you can finally update Ditto’s data:
+Once it’s done, you can update Ditto’s data:
 
 ``` bash
 cd ~/ditto
 rm -r ./data
 poetry install
-poetry run ditto clone --source http://localhost/ --destination ./data
+poetry run ditto clone --src-url http://localhost/ --dest-dir ./data
 poetry run ditto analyze --api-dir ./data/api --schema-dir ./data/schema
 ```
 
 This will crawl your local instance of PokeAPI, copy all the data to
-./data, and regenerate the schema. Once that's finished, you can serve
-the freshly updated data!
+./data, and regenerate the schema.
+
+Once that's finished, you can serve the freshly updated data!
 
 ``` bash
-poetry run ditto serve --port 8080
-```
+poetry run ditto serve --port 8080 --base-url http://localhost:8080
+```

pokeapi_ditto/__init__.py

@@ -1 +1 @@
-__version__ = "0.2.0"
+__version__ = "0.3.0"

pokeapi_ditto/analyze.py → pokeapi_ditto/commands/analyze.py

@@ -7,28 +7,14 @@
 
 from genson import SchemaBuilder
 
+from pokeapi_ditto.common import from_dir
 
-def _from_dir(target_dir: str):
-    target_dir = os.path.abspath(target_dir)
 
-    def func_decorator(func: callable):
-        def func_wrapper(*args, **kwargs):
-            cwd = os.getcwd()
-            os.chdir(target_dir)
-            result = func(*args, **kwargs)
-            os.chdir(cwd)
-            return result
-
-        return func_wrapper
-
-    return func_decorator
-
-
-def do_analyze(data_dir: str, schema_dir: str):
+def do_analyze(api_dir: str, schema_dir: str):
     if not Path(schema_dir).exists():
         Path(schema_dir).mkdir(parents=True)
 
-    @_from_dir(data_dir)
+    @from_dir(api_dir)
     def get_schema_paths() -> List[Path]:
         return sorted(
             {
@@ -37,22 +23,22 @@ def get_schema_paths() -> List[Path]:
             }
         )
 
-    @_from_dir(data_dir)
+    @from_dir(api_dir)
     def gen_single_schema(path: Path) -> SchemaBuilder:
         glob_exp = os.path.join(
             *["*" if part == "$id" else part for part in path.parts]
         )
-        print(os.path.join(*Path(data_dir).parts, glob_exp))
         file_names = glob.iglob(glob_exp, recursive=True)
         schema = SchemaBuilder()
         for file_name in file_names:
             with open(file_name) as f:
                 schema.add_object(json.load(f))
         return schema
 
-    @_from_dir(schema_dir)
+    @from_dir(schema_dir)
     def gen_schemas(paths: List[Path]):
         for path in paths:
+            print(Path(schema_dir).joinpath(path))
             if not path.parent.exists():
                 os.makedirs(path.parent)
             schema = gen_single_schema(path)

pokeapi_ditto/clone.py → pokeapi_ditto/commands/clone.py

@@ -4,29 +4,31 @@
 
 import requests
 
+from pokeapi_ditto.common import BASE_URL_PLACEHOLDER
+
+
+def do_clone(src_url, dest_dir):
+    if not src_url.endswith("/"):
+        src_url += "/"
+
+    if not dest_dir.endswith("/"):
+        dest_dir += "/"
 
-def do_clone(base_url, target_dir, replacement_url):
     def safe_open_w(file_name):
         os.makedirs(os.path.dirname(file_name), exist_ok=True)
         return open(file_name, "w")
 
     def print_json(data, file_name):
         transformed_data = json.dumps(data, indent=4, sort_keys=True)
-        transformed_data = transformed_data.replace(base_url, replacement_url)
+        transformed_data = transformed_data.replace(src_url, BASE_URL_PLACEHOLDER + "/")
         print(transformed_data, file=safe_open_w(file_name))
 
-    if not base_url.endswith("/"):
-        base_url += "/"
-
-    if not target_dir.endswith("/"):
-        target_dir += "/"
-
     # Root
 
-    url = base_url + "api/v2/"
+    url = src_url + "api/v2/"
     endpoints = requests.get(url)
 
-    path = target_dir + url.replace(base_url, "") + "index.json"
+    path = dest_dir + url.replace(src_url, "") + "index.json"
     print(path)
     print_json(endpoints.json(), path)
 
@@ -41,14 +43,14 @@ def print_json(data, file_name):
         # Full index
         url = endpoint + "?limit=" + count
         resource_list = requests.get(url)
-        path = target_dir + endpoint.replace(base_url, "") + "index.json"
+        path = dest_dir + endpoint.replace(src_url, "") + "index.json"
         print(path)
         print_json(resource_list.json(), path)
 
         # All resources
         for resourceSummary in resource_list.json()["results"]:
             resource_url = resourceSummary["url"]
-            path = target_dir + resource_url.replace(base_url, "") + "index.json"
+            path = dest_dir + resource_url.replace(src_url, "") + "index.json"
 
             if not os.path.isfile(path):
                 print(path)
@@ -57,7 +59,7 @@ def print_json(data, file_name):
 
             if endpoint.endswith("/pokemon/"):
                 resource_url += "encounters/"
-                path = target_dir + resource_url.replace(base_url, "") + "index.json"
+                path = dest_dir + resource_url.replace(src_url, "") + "index.json"
                 if not os.path.isfile(path):
                     print(path)
                     resource = requests.get(resource_url)